test_runner.py 17 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422
  1. #!/usr/bin/env vpython3
  2. #
  3. # Copyright 2018 The Chromium Authors. All rights reserved.
  4. # Use of this source code is governed by a BSD-style license that can be
  5. # found in the LICENSE file.
  6. """Deploys and runs a test package on a Fuchsia target."""
  7. import argparse
  8. import logging
  9. import os
  10. import shutil
  11. import sys
  12. import tempfile
  13. import ffx_session
  14. from common_args import AddCommonArgs, AddTargetSpecificArgs, \
  15. ConfigureLogging, GetDeploymentTargetForArgs
  16. from net_test_server import SetupTestServer
  17. from run_test_package import RunTestPackage, RunTestPackageArgs
  18. from runner_exceptions import HandleExceptionAndReturnExitCode
  19. sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__),
  20. 'test')))
  21. from compatible_utils import map_filter_file_to_package_file
  22. DEFAULT_TEST_SERVER_CONCURRENCY = 4
  23. TEST_DATA_DIR = '/tmp'
  24. TEST_FILTER_PATH = TEST_DATA_DIR + '/test_filter.txt'
  25. TEST_LLVM_PROFILE_DIR = 'llvm-profile'
  26. TEST_PERF_RESULT_FILE = 'test_perf_summary.json'
  27. TEST_RESULT_FILE = 'test_summary.json'
  28. TEST_REALM_NAME = 'chromium_tests'
  29. FILTER_DIR = 'testing/buildbot/filters'
  30. class TestOutputs(object):
  31. """An abstract base class for extracting outputs generated by a test."""
  32. def __init__(self):
  33. pass
  34. def __enter__(self):
  35. return self
  36. def __exit__(self, exc_type, exc_val, exc_tb):
  37. return False
  38. def GetFfxSession(self):
  39. raise NotImplementedError()
  40. def GetDevicePath(self, path):
  41. """Returns an absolute device-local variant of a path."""
  42. raise NotImplementedError()
  43. def GetFile(self, glob, destination):
  44. """Places all files/directories matched by a glob into a destination."""
  45. raise NotImplementedError()
  46. def GetCoverageProfiles(self, destination):
  47. """Places all coverage files from the target into a destination."""
  48. raise NotImplementedError()
  49. class TargetTestOutputs(TestOutputs):
  50. """A TestOutputs implementation for CFv1 tests, where tests emit files into
  51. /tmp that are retrieved from the device via ssh."""
  52. def __init__(self, target, package_name, test_realms):
  53. super(TargetTestOutputs, self).__init__()
  54. self._target = target
  55. self._package_name = package_name
  56. self._test_realms = test_realms
  57. def GetFfxSession(self):
  58. return None # ffx is not used to run CFv1 tests.
  59. def GetDevicePath(self, path):
  60. return TEST_DATA_DIR + '/' + path
  61. def GetFile(self, glob, destination):
  62. """Places all files/directories matched by a glob into a destination."""
  63. self._target.GetFile(self.GetDevicePath(glob),
  64. destination,
  65. for_package=self._package_name,
  66. for_realms=self._test_realms)
  67. def GetCoverageProfiles(self, destination):
  68. # Copy all the files in the profile directory. /* is used instead of
  69. # recursively copying due to permission issues for the latter.
  70. self._target.GetFile(self.GetDevicePath(TEST_LLVM_PROFILE_DIR + '/*'),
  71. destination, None, None)
  72. class CustomArtifactsTestOutputs(TestOutputs):
  73. """A TestOutputs implementation for CFv2 tests, where tests emit files into
  74. /custom_artifacts that are retrieved from the device automatically via ffx."""
  75. def __init__(self, target):
  76. super(CustomArtifactsTestOutputs, self).__init__()
  77. self._target = target
  78. self._ffx_session_context = ffx_session.FfxSession(target._log_manager)
  79. self._ffx_session = None
  80. def __enter__(self):
  81. self._ffx_session = self._ffx_session_context.__enter__()
  82. return self
  83. def __exit__(self, exc_type, exc_val, exc_tb):
  84. self._ffx_session = None
  85. self._ffx_session_context.__exit__(exc_type, exc_val, exc_tb)
  86. return False
  87. def GetFfxSession(self):
  88. assert self._ffx_session
  89. return self._ffx_session
  90. def GetDevicePath(self, path):
  91. return '/custom_artifacts/' + path
  92. def GetOutputDirectory(self):
  93. return self._ffx_session.get_output_dir()
  94. def GetFile(self, glob, destination):
  95. """Places all files/directories matched by a glob into a destination."""
  96. directory = self._ffx_session.get_custom_artifact_directory()
  97. if not directory:
  98. logger.error(
  99. 'Failed to parse custom artifact directory from test summary output '
  100. 'files. Not copying %s from the device', glob)
  101. return
  102. shutil.copy(os.path.join(directory, glob), destination)
  103. def GetCoverageProfiles(self, destination):
  104. # Copy all the files in the profile directory.
  105. # TODO(https://fxbug.dev/77634): Switch to ffx-based extraction once it is
  106. # implemented.
  107. self._target.GetFile(
  108. '/tmp/test_manager:0/children/debug_data:0/data/' +
  109. TEST_LLVM_PROFILE_DIR + '/*', destination)
  110. def MakeTestOutputs(component_version, target, package_name, test_realms):
  111. if component_version == '2':
  112. return CustomArtifactsTestOutputs(target)
  113. return TargetTestOutputs(target, package_name, test_realms)
  114. def AddTestExecutionArgs(arg_parser):
  115. test_args = arg_parser.add_argument_group('testing',
  116. 'Test execution arguments')
  117. test_args.add_argument('--gtest_filter',
  118. help='GTest filter to use in place of any default.')
  119. test_args.add_argument(
  120. '--gtest_repeat',
  121. help='GTest repeat value to use. This also disables the '
  122. 'test launcher timeout.')
  123. test_args.add_argument(
  124. '--test-launcher-retry-limit',
  125. help='Number of times that test suite will retry failing '
  126. 'tests. This is multiplicative with --gtest_repeat.')
  127. test_args.add_argument('--test-launcher-print-test-stdio',
  128. choices=['auto', 'always', 'never'],
  129. help='Controls when full test output is printed.'
  130. 'auto means to print it when the test failed.')
  131. test_args.add_argument('--test-launcher-shard-index',
  132. type=int,
  133. default=os.environ.get('GTEST_SHARD_INDEX'),
  134. help='Index of this instance amongst swarming shards.')
  135. test_args.add_argument('--test-launcher-total-shards',
  136. type=int,
  137. default=os.environ.get('GTEST_TOTAL_SHARDS'),
  138. help='Total number of swarming shards of this suite.')
  139. test_args.add_argument('--gtest_break_on_failure',
  140. action='store_true',
  141. default=False,
  142. help='Should GTest break on failure; useful with '
  143. '--gtest_repeat.')
  144. test_args.add_argument('--single-process-tests',
  145. action='store_true',
  146. default=False,
  147. help='Runs the tests and the launcher in the same '
  148. 'process. Useful for debugging.')
  149. test_args.add_argument('--test-launcher-batch-limit',
  150. type=int,
  151. help='Sets the limit of test batch to run in a single '
  152. 'process.')
  153. # --test-launcher-filter-file is specified relative to --out-dir,
  154. # so specifying type=os.path.* will break it.
  155. test_args.add_argument(
  156. '--test-launcher-filter-file',
  157. default=None,
  158. help='Filter file(s) passed to target test process. Use ";" to separate '
  159. 'multiple filter files ')
  160. test_args.add_argument('--test-launcher-jobs',
  161. type=int,
  162. help='Sets the number of parallel test jobs.')
  163. test_args.add_argument('--test-launcher-summary-output',
  164. help='Where the test launcher will output its json.')
  165. test_args.add_argument('--enable-test-server',
  166. action='store_true',
  167. default=False,
  168. help='Enable Chrome test server spawner.')
  169. test_args.add_argument(
  170. '--test-launcher-bot-mode',
  171. action='store_true',
  172. default=False,
  173. help='Informs the TestLauncher to that it should enable '
  174. 'special allowances for running on a test bot.')
  175. test_args.add_argument('--isolated-script-test-output',
  176. help='If present, store test results on this path.')
  177. test_args.add_argument(
  178. '--isolated-script-test-perf-output',
  179. help='If present, store chartjson results on this path.')
  180. test_args.add_argument('--use-run',
  181. dest='use_run_test_component',
  182. default=True,
  183. action='store_false',
  184. help='Run the test package using run rather than '
  185. 'hermetically using run-test-component.')
  186. test_args.add_argument(
  187. '--code-coverage',
  188. default=False,
  189. action='store_true',
  190. help='Gather code coverage information and place it in '
  191. 'the output directory.')
  192. test_args.add_argument('--code-coverage-dir',
  193. default=os.getcwd(),
  194. help='Directory to place code coverage information. '
  195. 'Only relevant when --code-coverage set to true. '
  196. 'Defaults to current directory.')
  197. test_args.add_argument('--gtest_also_run_disabled_tests',
  198. default=False,
  199. action='store_true',
  200. help='Run tests prefixed with DISABLED_')
  201. test_args.add_argument('child_args',
  202. nargs='*',
  203. help='Arguments for the test process.')
  204. test_args.add_argument('--use-vulkan',
  205. help='\'native\', \'swiftshader\' or \'none\'.')
  206. def main():
  207. parser = argparse.ArgumentParser()
  208. AddTestExecutionArgs(parser)
  209. AddCommonArgs(parser)
  210. AddTargetSpecificArgs(parser)
  211. args = parser.parse_args()
  212. # Flag out_dir is required for tests launched with this script.
  213. if not args.out_dir:
  214. raise ValueError("out-dir must be specified.")
  215. if args.component_version == "2":
  216. args.use_run_test_component = False
  217. if (args.code_coverage and args.component_version != "2"
  218. and not args.use_run_test_component):
  219. if args.enable_test_server:
  220. # TODO(1254563): Tests that need access to the test server cannot be run
  221. # as test component under CFv1. Because code coverage requires it, force
  222. # the test to run as a test component. It is expected that test that tries
  223. # to use the external test server will fail.
  224. args.use_run_test_component = True
  225. else:
  226. raise ValueError('Collecting code coverage info requires using '
  227. 'run-test-component.')
  228. ConfigureLogging(args)
  229. child_args = []
  230. if args.test_launcher_shard_index != None:
  231. child_args.append(
  232. '--test-launcher-shard-index=%d' % args.test_launcher_shard_index)
  233. if args.test_launcher_total_shards != None:
  234. child_args.append(
  235. '--test-launcher-total-shards=%d' % args.test_launcher_total_shards)
  236. if args.single_process_tests:
  237. child_args.append('--single-process-tests')
  238. if args.test_launcher_bot_mode:
  239. child_args.append('--test-launcher-bot-mode')
  240. if args.test_launcher_batch_limit:
  241. child_args.append('--test-launcher-batch-limit=%d' %
  242. args.test_launcher_batch_limit)
  243. # Only set --test-launcher-jobs if the caller specifies it, in general.
  244. # If the caller enables the test-server then we need to launch the right
  245. # number of instances to match the maximum number of parallel test jobs, so
  246. # in that case we set --test-launcher-jobs based on the number of CPU cores
  247. # specified for the emulator to use.
  248. test_concurrency = None
  249. if args.test_launcher_jobs:
  250. test_concurrency = args.test_launcher_jobs
  251. elif args.enable_test_server:
  252. if args.device == 'device':
  253. test_concurrency = DEFAULT_TEST_SERVER_CONCURRENCY
  254. else:
  255. test_concurrency = args.cpu_cores
  256. if test_concurrency:
  257. child_args.append('--test-launcher-jobs=%d' % test_concurrency)
  258. if args.test_launcher_print_test_stdio:
  259. child_args.append('--test-launcher-print-test-stdio=%s' %
  260. args.test_launcher_print_test_stdio)
  261. if args.gtest_filter:
  262. child_args.append('--gtest_filter=' + args.gtest_filter)
  263. if args.gtest_repeat:
  264. child_args.append('--gtest_repeat=' + args.gtest_repeat)
  265. child_args.append('--test-launcher-timeout=-1')
  266. if args.test_launcher_retry_limit:
  267. child_args.append(
  268. '--test-launcher-retry-limit=' + args.test_launcher_retry_limit)
  269. if args.gtest_break_on_failure:
  270. child_args.append('--gtest_break_on_failure')
  271. if args.gtest_also_run_disabled_tests:
  272. child_args.append('--gtest_also_run_disabled_tests')
  273. if args.child_args:
  274. child_args.extend(args.child_args)
  275. test_realms = []
  276. if args.use_run_test_component:
  277. test_realms = [TEST_REALM_NAME]
  278. if args.use_vulkan:
  279. child_args.append('--use-vulkan=' + args.use_vulkan)
  280. elif args.target_cpu == 'x64':
  281. # TODO(crbug.com/1261646) Remove once Vulkan is enabled by default.
  282. child_args.append('--use-vulkan=native')
  283. else:
  284. # Use swiftshader on arm64 by default because arm64 bots currently
  285. # don't support Vulkan emulation.
  286. child_args.append('--use-vulkan=swiftshader')
  287. try:
  288. with GetDeploymentTargetForArgs(args) as target, \
  289. MakeTestOutputs(args.component_version,
  290. target,
  291. args.package_name,
  292. test_realms) as test_outputs:
  293. if args.test_launcher_summary_output:
  294. child_args.append('--test-launcher-summary-output=' +
  295. test_outputs.GetDevicePath(TEST_RESULT_FILE))
  296. if args.isolated_script_test_output:
  297. child_args.append('--isolated-script-test-output=' +
  298. test_outputs.GetDevicePath(TEST_RESULT_FILE))
  299. if args.isolated_script_test_perf_output:
  300. child_args.append('--isolated-script-test-perf-output=' +
  301. test_outputs.GetDevicePath(TEST_PERF_RESULT_FILE))
  302. target.Start()
  303. target.StartSystemLog(args.package)
  304. if args.test_launcher_filter_file:
  305. if args.component_version == "2":
  306. # TODO(crbug.com/1279803): Until one can send file to the device when
  307. # running a test, filter files must be read from the test package.
  308. test_launcher_filter_files = map(
  309. map_filter_file_to_package_file,
  310. args.test_launcher_filter_file.split(';'))
  311. child_args.append('--test-launcher-filter-file=' +
  312. ';'.join(test_launcher_filter_files))
  313. else:
  314. test_launcher_filter_files = args.test_launcher_filter_file.split(';')
  315. with tempfile.NamedTemporaryFile('a+b') as combined_filter_file:
  316. for filter_file in test_launcher_filter_files:
  317. with open(filter_file, 'rb') as f:
  318. combined_filter_file.write(f.read())
  319. combined_filter_file.seek(0)
  320. target.PutFile(combined_filter_file.name,
  321. TEST_FILTER_PATH,
  322. for_package=args.package_name,
  323. for_realms=test_realms)
  324. child_args.append('--test-launcher-filter-file=' + TEST_FILTER_PATH)
  325. test_server = None
  326. if args.enable_test_server:
  327. assert test_concurrency
  328. (test_server,
  329. spawner_url_base) = SetupTestServer(target, test_concurrency)
  330. child_args.append('--remote-test-server-spawner-url-base=' +
  331. spawner_url_base)
  332. run_package_args = RunTestPackageArgs.FromCommonArgs(args)
  333. if args.use_run_test_component:
  334. run_package_args.test_realm_label = TEST_REALM_NAME
  335. run_package_args.use_run_test_component = True
  336. if args.component_version == "2":
  337. run_package_args.output_directory = test_outputs.GetOutputDirectory()
  338. returncode = RunTestPackage(target, test_outputs.GetFfxSession(),
  339. args.package, args.package_name,
  340. args.component_version, child_args,
  341. run_package_args)
  342. if test_server:
  343. test_server.Stop()
  344. if args.code_coverage:
  345. test_outputs.GetCoverageProfiles(args.code_coverage_dir)
  346. if args.test_launcher_summary_output:
  347. test_outputs.GetFile(TEST_RESULT_FILE,
  348. args.test_launcher_summary_output)
  349. if args.isolated_script_test_output:
  350. test_outputs.GetFile(TEST_RESULT_FILE, args.isolated_script_test_output)
  351. if args.isolated_script_test_perf_output:
  352. test_outputs.GetFile(TEST_PERF_RESULT_FILE,
  353. args.isolated_script_test_perf_output)
  354. return returncode
  355. except:
  356. return HandleExceptionAndReturnExitCode()
  357. if __name__ == '__main__':
  358. sys.exit(main())