wpt_common.py 14 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356
  1. # Copyright 2020 The Chromium Authors. All rights reserved.
  2. # Use of this source code is governed by a BSD-style license that can be
  3. # found in the LICENSE file.
  4. import argparse
  5. import glob
  6. import logging
  7. import os
  8. import sys
  9. # Add src/testing/ into sys.path for importing common without pylint errors.
  10. sys.path.append(
  11. os.path.abspath(os.path.join(os.path.dirname(__file__), os.path.pardir)))
  12. from scripts import common
  13. BLINK_TOOLS_DIR = os.path.join(common.SRC_DIR, 'third_party', 'blink', 'tools')
  14. CATAPULT_DIR = os.path.join(common.SRC_DIR, 'third_party', 'catapult')
  15. OUT_DIR = os.path.join(common.SRC_DIR, "out", "{}")
  16. DEFAULT_ISOLATED_SCRIPT_TEST_OUTPUT = os.path.join(OUT_DIR, "results.json")
  17. TYP_DIR = os.path.join(CATAPULT_DIR, 'third_party', 'typ')
  18. WEB_TESTS_DIR = os.path.normpath(
  19. os.path.join(BLINK_TOOLS_DIR, os.pardir, 'web_tests'))
  20. if BLINK_TOOLS_DIR not in sys.path:
  21. sys.path.append(BLINK_TOOLS_DIR)
  22. if TYP_DIR not in sys.path:
  23. sys.path.append(TYP_DIR)
  24. from blinkpy.common.host import Host
  25. from blinkpy.common.path_finder import PathFinder
  26. logger = logging.getLogger(__name__)
  27. # pylint: disable=super-with-arguments
  28. class BaseWptScriptAdapter(common.BaseIsolatedScriptArgsAdapter):
  29. """The base class for script adapters that use wptrunner to execute web
  30. platform tests. This contains any code shared between these scripts, such
  31. as integrating output with the results viewer. Subclasses contain other
  32. (usually platform-specific) logic."""
  33. def __init__(self, host=None):
  34. super(BaseWptScriptAdapter, self).__init__()
  35. self._parser = self._override_options(self._parser)
  36. if not host:
  37. host = Host()
  38. self.host = host
  39. self.fs = host.filesystem
  40. self.path_finder = PathFinder(self.fs)
  41. self.port = host.port_factory.get()
  42. self.wptreport = None
  43. self._include_filename = None
  44. self.layout_test_results_subdir = 'layout-test-results'
  45. @property
  46. def wpt_binary(self):
  47. default_wpt_binary = os.path.join(
  48. common.SRC_DIR, "third_party", "wpt_tools", "wpt", "wpt")
  49. return os.environ.get("WPT_BINARY", default_wpt_binary)
  50. @property
  51. def wpt_root_dir(self):
  52. return self.path_finder.path_from_web_tests(
  53. self.path_finder.wpt_prefix())
  54. @property
  55. def output_directory(self):
  56. return self.path_finder.path_from_chromium_base('out',
  57. self.options.target)
  58. @property
  59. def mojo_js_directory(self):
  60. return self.fs.join(self.output_directory, 'gen')
  61. def add_extra_arguments(self, parser):
  62. parser.add_argument(
  63. '-t',
  64. '--target',
  65. default='Release',
  66. help='Target build subdirectory under //out')
  67. parser.add_argument(
  68. '--default-exclude',
  69. action='store_true',
  70. help=('Only run the tests explicitly given in arguments '
  71. '(can run no tests, which will exit with code 0)'))
  72. self.add_mode_arguments(parser)
  73. self.add_output_arguments(parser)
  74. def add_mode_arguments(self, parser):
  75. group = parser.add_argument_group(
  76. 'Mode',
  77. 'Options for wptrunner modes other than running tests.')
  78. # We provide an option to show wptrunner's help here because the 'wpt'
  79. # executable may be inaccessible from the user's PATH. The top-level
  80. # 'wpt' command also needs to have virtualenv disabled.
  81. group.add_argument(
  82. '--wpt-help',
  83. action='store_true',
  84. help='Show the wptrunner help message and exit')
  85. return group
  86. def add_output_arguments(self, parser):
  87. group = parser.add_argument_group(
  88. 'Output Logging',
  89. 'Options for controlling logging behavior.')
  90. group.add_argument(
  91. '--log-wptreport',
  92. nargs='?',
  93. # We cannot provide a default, since the default filename depends on
  94. # the product, so we use this placeholder instead.
  95. const='',
  96. help=('Log a wptreport in JSON to the output directory '
  97. '(default filename: '
  98. 'wpt_reports_<product>_<shard-index>.json)'))
  99. group.add_argument(
  100. '-v',
  101. '--verbose',
  102. action='count',
  103. default=0,
  104. help='Increase verbosity')
  105. return group
  106. def _override_options(self, base_parser):
  107. """Create a parser that overrides existing options.
  108. `argument.ArgumentParser` can extend other parsers and override their
  109. options, with the caveat that the child parser only inherits options
  110. that the parent had at the time of the child's initialization. There is
  111. not a clean way to add option overrides in `add_extra_arguments`, where
  112. the provided parser is only passed up the inheritance chain, so we add
  113. overridden options here at the very end.
  114. See Also:
  115. https://docs.python.org/3/library/argparse.html#parents
  116. """
  117. parser = argparse.ArgumentParser(
  118. parents=[base_parser],
  119. # Allow overriding existing options in the parent parser.
  120. conflict_handler='resolve',
  121. epilog=('All unrecognized arguments are passed through '
  122. "to wptrunner. Use '--wpt-help' to see wptrunner's usage."),
  123. )
  124. parser.add_argument(
  125. '--isolated-script-test-repeat',
  126. '--repeat',
  127. '--gtest_repeat',
  128. metavar='REPEAT',
  129. type=int,
  130. default=1,
  131. help='Number of times to run the tests')
  132. parser.add_argument(
  133. '--isolated-script-test-launcher-retry-limit',
  134. '--test-launcher-retry-limit',
  135. metavar='LIMIT',
  136. type=int,
  137. default=0,
  138. help='Maximum number of times to rerun a failed test')
  139. # `--gtest_filter` and `--isolated-script-test-filter` have slightly
  140. # different formats and behavior, so keep them as separate options.
  141. # See: crbug/1316164#c4
  142. return parser
  143. def maybe_set_default_isolated_script_test_output(self):
  144. if self.options.isolated_script_test_output:
  145. return
  146. default_value = DEFAULT_ISOLATED_SCRIPT_TEST_OUTPUT.format(
  147. self.options.target)
  148. print("--isolated-script-test-output not set, defaulting to %s" %
  149. default_value)
  150. self.options.isolated_script_test_output = default_value
  151. def generate_test_output_args(self, output):
  152. return ['--log-chromium=%s' % output]
  153. def _resolve_tests_from_isolate_filter(self, test_filter):
  154. """Resolve an isolated script-style filter string into lists of tests.
  155. Arguments:
  156. test_filter (str): Glob patterns delimited by double colons ('::').
  157. The glob is prefixed with '-' to indicate that tests matching
  158. the pattern should not run. Assume a valid wpt name cannot start
  159. with '-'.
  160. Returns:
  161. tuple[list[str], list[str]]: Tests to include and exclude,
  162. respectively.
  163. """
  164. included_tests, excluded_tests = [], []
  165. for pattern in common.extract_filter_list(test_filter):
  166. test_group = included_tests
  167. if pattern.startswith('-'):
  168. test_group, pattern = excluded_tests, pattern[1:]
  169. pattern_on_disk = self.fs.join(
  170. self.wpt_root_dir,
  171. self.path_finder.strip_wpt_path(pattern),
  172. )
  173. test_group.extend(glob.glob(pattern_on_disk))
  174. return included_tests, excluded_tests
  175. def generate_test_filter_args(self, test_filter_str):
  176. included_tests, excluded_tests = \
  177. self._resolve_tests_from_isolate_filter(test_filter_str)
  178. include_file, self._include_filename = self.fs.open_text_tempfile()
  179. with include_file:
  180. for test in included_tests:
  181. include_file.write(test)
  182. include_file.write('\n')
  183. wpt_args = ['--include-file=%s' % self._include_filename]
  184. for test in excluded_tests:
  185. wpt_args.append('--exclude=%s' % test)
  186. return wpt_args
  187. def generate_test_repeat_args(self, repeat_count):
  188. return ['--repeat=%d' % repeat_count]
  189. # pylint: disable=unused-argument
  190. def generate_test_launcher_retry_limit_args(self, retry_limit):
  191. # TODO(crbug/1306222): wptrunner currently cannot rerun individual
  192. # failed tests, so this flag is accepted but not used.
  193. return []
  194. def generate_sharding_args(self, total_shards, shard_index):
  195. return ['--total-chunks=%d' % total_shards,
  196. # shard_index is 0-based but WPT's this-chunk to be 1-based
  197. '--this-chunk=%d' % (shard_index + 1),
  198. # The default sharding strategy is to shard by directory. But
  199. # we want to hash each test to determine which shard runs it.
  200. # This allows running individual directories that have few
  201. # tests across many shards.
  202. '--chunk-type=hash']
  203. def parse_args(self, args=None):
  204. super(BaseWptScriptAdapter, self).parse_args(args)
  205. if self.options.wpt_help:
  206. self._show_wpt_help()
  207. # Update the output directory and wptreport filename to defaults if not
  208. # set. We cannot provide CLI option defaults because they depend on
  209. # other options ('--target' and '--product').
  210. self.maybe_set_default_isolated_script_test_output()
  211. report = self.options.log_wptreport
  212. if report is not None:
  213. if not report:
  214. report = self._default_wpt_report()
  215. self.wptreport = self.fs.join(self.fs.dirname(self.wpt_output),
  216. report)
  217. @property
  218. def wpt_output(self):
  219. return self.options.isolated_script_test_output
  220. def _show_wpt_help(self):
  221. command = [
  222. self.select_python_executable(),
  223. ]
  224. command.extend(self._wpt_run_args)
  225. command.extend(['--help'])
  226. exit_code = common.run_command(command)
  227. self.parser.exit(exit_code)
  228. @property
  229. def _wpt_run_args(self):
  230. """The start of a 'wpt run' command."""
  231. return [
  232. self.wpt_binary,
  233. # Use virtualenv packages installed by vpython, not wpt.
  234. '--venv=%s' % self.path_finder.chromium_base(),
  235. '--skip-venv-setup',
  236. 'run',
  237. ]
  238. @property
  239. def rest_args(self):
  240. unknown_args = super(BaseWptScriptAdapter, self).rest_args
  241. rest_args = list(self._wpt_run_args)
  242. rest_args.extend([
  243. # By default, wpt will treat unexpected passes as errors, so we
  244. # disable that to be consistent with Chromium CI.
  245. '--no-fail-on-unexpected-pass',
  246. '--no-pause-after-test',
  247. '--no-capture-stdio',
  248. '--no-manifest-download',
  249. '--tests=%s' % self.wpt_root_dir,
  250. '--mojojs-path=%s' % self.mojo_js_directory,
  251. ])
  252. if self.options.default_exclude:
  253. rest_args.extend(['--default-exclude'])
  254. if self.wptreport:
  255. rest_args.extend(['--log-wptreport', self.wptreport])
  256. if self.options.verbose >= 3:
  257. rest_args.extend([
  258. '--log-mach=-',
  259. '--log-mach-level=debug',
  260. '--log-mach-verbose',
  261. ])
  262. if self.options.verbose >= 4:
  263. rest_args.extend([
  264. '--webdriver-arg=--verbose',
  265. '--webdriver-arg="--log-path=-"',
  266. ])
  267. rest_args.append(self.wpt_product_name())
  268. # We pass through unknown args as late as possible so that they can
  269. # override earlier options. It also allows users to pass test names as
  270. # positional args, which must not have option strings between them.
  271. for unknown_arg in unknown_args:
  272. # crbug/1274933#c14: Some developers had used the end-of-options
  273. # marker '--' to pass through arguments to wptrunner.
  274. # crrev.com/c/3573284 makes this no longer necessary.
  275. if unknown_arg == '--':
  276. logger.warning(
  277. 'Unrecognized options will automatically fall through '
  278. 'to wptrunner.')
  279. logger.warning(
  280. "There is no need to use the end-of-options marker '--'.")
  281. else:
  282. rest_args.append(unknown_arg)
  283. return rest_args
  284. def process_and_upload_results(self):
  285. command = [
  286. self.select_python_executable(),
  287. os.path.join(BLINK_TOOLS_DIR, 'wpt_process_results.py'),
  288. '--target',
  289. self.options.target,
  290. '--web-tests-dir',
  291. WEB_TESTS_DIR,
  292. '--artifacts-dir',
  293. os.path.join(os.path.dirname(self.wpt_output),
  294. self.layout_test_results_subdir),
  295. '--wpt-results',
  296. self.wpt_output,
  297. ]
  298. if self.options.verbose:
  299. command.append('--verbose')
  300. if self.wptreport:
  301. command.extend(['--wpt-report', self.wptreport])
  302. common.run_command(command)
  303. def clean_up_after_test_run(self):
  304. if self._include_filename:
  305. self.fs.remove(self._include_filename)
  306. def wpt_product_name(self):
  307. raise NotImplementedError
  308. def _default_wpt_report(self):
  309. product = self.wpt_product_name()
  310. shard_index = os.environ.get('GTEST_SHARD_INDEX')
  311. if shard_index is not None:
  312. return 'wpt_reports_%s_%02d.json' % (product, int(shard_index))
  313. return 'wpt_reports_%s.json' % product