run_wpt_tests.py 33 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901
  1. #!/usr/bin/env vpython3
  2. # Copyright 2018 The Chromium Authors. All rights reserved.
  3. # Use of this source code is governed by a BSD-style license that can be
  4. # found in the LICENSE file.
  5. """Run web platform tests for Chromium-related products."""
  6. import argparse
  7. import contextlib
  8. import json
  9. import logging
  10. import os
  11. import shutil
  12. import sys
  13. import tempfile
  14. import wpt_common
  15. # Add src/testing/ into sys.path for importing common without pylint errors.
  16. sys.path.append(
  17. os.path.abspath(os.path.join(os.path.dirname(__file__), os.path.pardir)))
  18. from scripts import common
  19. logger = logging.getLogger(__name__)
  20. SRC_DIR = os.path.abspath(
  21. os.path.join(os.path.dirname(__file__), os.pardir, os.pardir))
  22. BUILD_ANDROID = os.path.join(SRC_DIR, 'build', 'android')
  23. BLINK_TOOLS_DIR = os.path.join(
  24. SRC_DIR, 'third_party', 'blink', 'tools')
  25. UPSTREAM_GIT_URL = 'https://github.com/web-platform-tests/wpt.git'
  26. if BLINK_TOOLS_DIR not in sys.path:
  27. sys.path.append(BLINK_TOOLS_DIR)
  28. if BUILD_ANDROID not in sys.path:
  29. sys.path.append(BUILD_ANDROID)
  30. from blinkpy.common.path_finder import PathFinder
  31. from blinkpy.web_tests.port.android import (
  32. PRODUCTS,
  33. PRODUCTS_TO_EXPECTATION_FILE_PATHS,
  34. ANDROID_WEBLAYER,
  35. ANDROID_WEBVIEW,
  36. CHROME_ANDROID,
  37. ANDROID_DISABLED_TESTS,
  38. )
  39. try:
  40. # This import adds `devil` to `sys.path`.
  41. import devil_chromium
  42. from devil import devil_env
  43. from devil.utils.parallelizer import SyncParallelizer
  44. from devil.android import apk_helper
  45. from devil.android import device_utils
  46. from devil.android.device_errors import CommandFailedError
  47. from devil.android.tools import webview_app
  48. from pylib.local.emulator import avd
  49. _ANDROID_ENABLED = True
  50. except ImportError:
  51. logger.warning('Android tools not found')
  52. _ANDROID_ENABLED = False
  53. def _make_pass_through_action(dest, map_arg=lambda arg: arg):
  54. class PassThroughAction(argparse.Action):
  55. def __init__(self, option_strings, dest, nargs=None, **kwargs):
  56. if nargs is not None and not isinstance(nargs, int):
  57. raise ValueError('nargs {} not supported for {}'.format(
  58. nargs, option_strings))
  59. super().__init__(option_strings, dest, nargs=nargs, **kwargs)
  60. def __call__(self, parser, namespace, values, option_string=None):
  61. if not option_string:
  62. return
  63. args = [option_string]
  64. if self.nargs is None:
  65. # Typically a single-arg option, but *not* wrapped in a list,
  66. # as is the case for `nargs=1`.
  67. args.append(str(values))
  68. else:
  69. args.extend(map(str, values))
  70. # Use the single-arg form of a long option. Easier to read with
  71. # option prefixing. Example:
  72. # --binary-arg=--enable-blink-features=Feature
  73. # instead of
  74. # --binary-arg=--enable-blink-features --binary-arg=Feature
  75. if len(args) == 2 and args[0].startswith('--'):
  76. args = ['%s=%s' % (args[0], args[1])]
  77. wpt_args = getattr(namespace, dest, [])
  78. wpt_args.extend(map(map_arg, args))
  79. setattr(namespace, dest, wpt_args)
  80. return PassThroughAction
  81. WPTPassThroughAction = _make_pass_through_action('wpt_args')
  82. BinaryPassThroughAction = _make_pass_through_action(
  83. 'wpt_args',
  84. lambda arg: '--binary-arg=%s' % arg)
  85. class WPTAdapter(wpt_common.BaseWptScriptAdapter):
  86. def __init__(self):
  87. self._metadata_dir = None
  88. self.temp_dir = os.path.join(tempfile.gettempdir(), "upstream_wpt")
  89. super().__init__()
  90. # Parent adapter adds extra arguments, so it is safe to parse the
  91. # arguments and set options here.
  92. try:
  93. self.parse_args()
  94. product_cls = _product_registry[self.options.product_name]
  95. self.product = product_cls(self.host,
  96. self.options,
  97. self.select_python_executable())
  98. except ValueError as exc:
  99. self._parser.error(str(exc))
  100. def parse_args(self, args=None):
  101. super().parse_args(args)
  102. if not hasattr(self.options, 'wpt_args'):
  103. self.options.wpt_args = []
  104. logging.basicConfig(
  105. level=self.log_level,
  106. # Align level name for easier reading.
  107. format='%(asctime)s [%(levelname)-8s] %(name)s: %(message)s',
  108. force=True)
  109. @property
  110. def wpt_binary(self):
  111. if self.options.use_upstream_wpt:
  112. return os.path.join(self.temp_dir, "wpt")
  113. return super().wpt_binary
  114. @property
  115. def wpt_root_dir(self):
  116. if self.options.use_upstream_wpt:
  117. return self.temp_dir
  118. return super().wpt_root_dir
  119. @property
  120. def rest_args(self):
  121. rest_args = super().rest_args
  122. rest_args.extend([
  123. '--webdriver-arg=--enable-chrome-logs',
  124. # TODO(crbug/1316055): Enable tombstone with '--stackwalk-binary'
  125. # and '--symbols-path'.
  126. '--headless',
  127. # Exclude webdriver tests for now. The CI runs them separately.
  128. '--exclude=webdriver',
  129. '--exclude=infrastructure/webdriver',
  130. '--binary-arg=--host-resolver-rules='
  131. 'MAP nonexistent.*.test ~NOTFOUND, MAP *.test 127.0.0.1',
  132. '--binary-arg=--enable-experimental-web-platform-features',
  133. '--binary-arg=--enable-blink-features=MojoJS,MojoJSTest',
  134. '--binary-arg=--enable-blink-test-features',
  135. '--binary-arg=--disable-field-trial-config',
  136. '--binary-arg=--enable-features='
  137. 'DownloadService<DownloadServiceStudy',
  138. '--binary-arg=--force-fieldtrials=DownloadServiceStudy/Enabled',
  139. '--binary-arg=--force-fieldtrial-params='
  140. 'DownloadServiceStudy.Enabled:start_up_delay_ms/0',
  141. ])
  142. rest_args.extend(self.product.wpt_args)
  143. # if metadata was created then add the metadata directory
  144. # to the list of wpt arguments
  145. if self._metadata_dir:
  146. rest_args.extend(['--metadata', self._metadata_dir])
  147. if self.options.test_filter:
  148. for pattern in self.options.test_filter.split(':'):
  149. rest_args.extend([
  150. '--include',
  151. self.path_finder.strip_wpt_path(pattern),
  152. ])
  153. rest_args.extend(self.options.wpt_args)
  154. return rest_args
  155. def _maybe_build_metadata(self):
  156. metadata_builder_cmd = [
  157. self.select_python_executable(),
  158. self.path_finder.path_from_blink_tools('build_wpt_metadata.py'),
  159. '--metadata-output-dir=%s' % self._metadata_dir,
  160. ]
  161. if self.options.ignore_default_expectations:
  162. metadata_builder_cmd += [ '--ignore-default-expectations' ]
  163. metadata_builder_cmd.extend(self.product.metadata_builder_args)
  164. return common.run_command(metadata_builder_cmd)
  165. @property
  166. def log_level(self):
  167. if self.options.verbose >= 2:
  168. return logging.DEBUG
  169. if self.options.verbose >= 1:
  170. return logging.INFO
  171. return logging.WARNING
  172. def run_test(self):
  173. with contextlib.ExitStack() as stack:
  174. tmp_dir = stack.enter_context(self.fs.mkdtemp())
  175. # Manually remove the temporary directory's contents recursively
  176. # after the tests complete. Otherwise, `mkdtemp()` raise an error.
  177. stack.callback(self.fs.rmtree, tmp_dir)
  178. stack.enter_context(self.product.test_env())
  179. self._metadata_dir = os.path.join(tmp_dir, 'metadata_dir')
  180. metadata_command_ret = self._maybe_build_metadata()
  181. if metadata_command_ret != 0:
  182. return metadata_command_ret
  183. # If there is no metadata then we need to create an
  184. # empty directory to pass to wptrunner
  185. if not os.path.exists(self._metadata_dir):
  186. os.makedirs(self._metadata_dir)
  187. if self.options.use_upstream_wpt:
  188. logger.info("Using upstream wpt, cloning to %s ..."
  189. % self.temp_dir)
  190. # check if directory exists, if it does remove it
  191. if os.path.isdir(self.temp_dir):
  192. shutil.rmtree(self.temp_dir, ignore_errors=True)
  193. # make a temp directory and git pull into it
  194. clone_cmd = ['git', 'clone', UPSTREAM_GIT_URL,
  195. self.temp_dir, '--depth=1']
  196. common.run_command(clone_cmd)
  197. return super().run_test()
  198. def do_post_test_run_tasks(self):
  199. self.process_and_upload_results()
  200. def clean_up_after_test_run(self):
  201. super().clean_up_after_test_run()
  202. # Avoid having a dangling reference to the temp directory
  203. # which was deleted
  204. self._metadata_dir = None
  205. if self.options.use_upstream_wpt:
  206. shutil.rmtree(self.temp_dir, ignore_errors=True)
  207. def add_extra_arguments(self, parser):
  208. super().add_extra_arguments(parser)
  209. parser.description = __doc__
  210. self.add_metadata_arguments(parser)
  211. self.add_binary_arguments(parser)
  212. self.add_test_arguments(parser)
  213. if _ANDROID_ENABLED:
  214. self.add_android_arguments(parser)
  215. parser.add_argument(
  216. '-p',
  217. '--product',
  218. dest='product_name',
  219. default='chrome',
  220. # The parser converts the value before checking if it is in choices,
  221. # so we avoid looking up the class right away.
  222. choices=sorted(_product_registry, key=len),
  223. help='Product (browser or browser component) to test.')
  224. parser.add_argument(
  225. '--webdriver-binary',
  226. help=('Path of the webdriver binary.'
  227. 'It needs to have the same major version '
  228. 'as the browser binary or APK.'))
  229. parser.add_argument(
  230. '--webdriver-arg',
  231. action=WPTPassThroughAction,
  232. help='WebDriver args.')
  233. parser.add_argument(
  234. '-j',
  235. '--processes',
  236. '--child-processes',
  237. type=lambda processes: max(0, int(processes)),
  238. default=1,
  239. help=('Number of drivers to start in parallel. (For Android, '
  240. 'this number is the number of emulators started.)'
  241. 'The actual number of devices tested may be higher '
  242. 'if physical devices are available.)'))
  243. parser.add_argument(
  244. '--use-upstream-wpt',
  245. action='store_true',
  246. help=('Use the upstream wpt, this tag will clone'
  247. 'the upstream github wpt to a temporary'
  248. 'directory and will use the binary and'
  249. 'tests from upstream')
  250. )
  251. def add_metadata_arguments(self, parser):
  252. group = parser.add_argument_group(
  253. 'Metadata Builder',
  254. 'Options for building WPT metadata from web test expectations.')
  255. group.add_argument(
  256. '--additional-expectations',
  257. metavar='EXPECTATIONS_FILE',
  258. action='append',
  259. default=[],
  260. help='Paths to additional test expectations files.')
  261. group.add_argument(
  262. '--ignore-default-expectations',
  263. action='store_true',
  264. help='Do not use the default set of TestExpectations files.')
  265. group.add_argument(
  266. '--ignore-browser-specific-expectations',
  267. action='store_true',
  268. default=False,
  269. help='Ignore browser-specific expectation files.')
  270. return group
  271. def add_binary_arguments(self, parser):
  272. group = parser.add_argument_group(
  273. 'Binary Configuration',
  274. 'Options for configuring the binary under test.')
  275. group.add_argument(
  276. '--enable-features',
  277. metavar='FEATURES',
  278. action=BinaryPassThroughAction,
  279. help='Chromium features to enable during testing.')
  280. group.add_argument(
  281. '--disable-features',
  282. metavar='FEATURES',
  283. action=BinaryPassThroughAction,
  284. help='Chromium features to disable during testing.')
  285. group.add_argument(
  286. '--force-fieldtrials',
  287. metavar='TRIALS',
  288. action=BinaryPassThroughAction,
  289. help='Force trials for Chromium features.')
  290. group.add_argument(
  291. '--force-fieldtrial-params',
  292. metavar='TRIAL_PARAMS',
  293. action=BinaryPassThroughAction,
  294. help='Force trial params for Chromium features.')
  295. return group
  296. def add_test_arguments(self, parser):
  297. group = parser.add_argument_group(
  298. 'Test Selection',
  299. 'Options for selecting tests to run.')
  300. group.add_argument(
  301. '--include',
  302. metavar='TEST_OR_DIR',
  303. action=WPTPassThroughAction,
  304. help=('Test(s) to run. Defaults to all tests, '
  305. "if '--default-exclude' not provided."))
  306. group.add_argument(
  307. '--include-file',
  308. action=WPTPassThroughAction,
  309. help='A file listing test(s) to run.')
  310. group.add_argument(
  311. '--test-filter',
  312. '--gtest_filter',
  313. metavar='TESTS_OR_DIRS',
  314. help='Colon-separated list of test names (URL prefixes).')
  315. return group
  316. def add_mode_arguments(self, parser):
  317. group = super().add_mode_arguments(parser)
  318. group.add_argument(
  319. '--list-tests',
  320. nargs=0,
  321. action=WPTPassThroughAction,
  322. help='List all tests that will run.')
  323. return group
  324. def add_output_arguments(self, parser):
  325. group = super().add_output_arguments(parser)
  326. group.add_argument(
  327. '--log-raw',
  328. metavar='RAW_REPORT_FILE',
  329. action=WPTPassThroughAction,
  330. help='Log raw report.')
  331. group.add_argument(
  332. '--log-html',
  333. metavar='HTML_REPORT_FILE',
  334. action=WPTPassThroughAction,
  335. help='Log html report.')
  336. group.add_argument(
  337. '--log-xunit',
  338. metavar='XUNIT_REPORT_FILE',
  339. action=WPTPassThroughAction,
  340. help='Log xunit report.')
  341. return group
  342. def add_android_arguments(self, parser):
  343. group = parser.add_argument_group(
  344. 'Android',
  345. 'Options for configuring Android devices and tooling.')
  346. add_emulator_args(group)
  347. group.add_argument(
  348. '--browser-apk',
  349. # Aliases for backwards compatibility.
  350. '--chrome-apk',
  351. '--system-webview-shell',
  352. '--weblayer-shell',
  353. help=('Path to the browser APK to install and run. '
  354. '(For WebView and WebLayer, this value is the shell. '
  355. 'Defaults to an on-device APK if not provided.)'))
  356. group.add_argument(
  357. '--webview-provider',
  358. help=('Path to a WebView provider APK to install. '
  359. '(WebView only.)'))
  360. group.add_argument(
  361. '--additional-apk',
  362. # Aliases for backwards compatibility.
  363. '--weblayer-support',
  364. action='append',
  365. default=[],
  366. help='Paths to additional APKs to install.')
  367. group.add_argument(
  368. '--release-channel',
  369. help='Install WebView from release channel. (WebView only.)')
  370. group.add_argument(
  371. '--package-name',
  372. # Aliases for backwards compatibility.
  373. '--chrome-package-name',
  374. help='Package name to run tests against.')
  375. group.add_argument(
  376. '--adb-binary',
  377. type=os.path.realpath,
  378. help='Path to adb binary to use.')
  379. return group
  380. def wpt_product_name(self):
  381. # `self.product` may not be set yet, so `self.product.name` is
  382. # unavailable. `self._options.product_name` may be an alias, so we need
  383. # to translate it into its wpt-accepted name.
  384. product_cls = _product_registry[self._options.product_name]
  385. return product_cls.name
  386. class Product:
  387. """A product (browser or browser component) that can run web platform tests.
  388. Attributes:
  389. name (str): The official wpt-accepted name of this product.
  390. aliases (list[str]): Human-friendly aliases for the official name.
  391. """
  392. name = ''
  393. aliases = []
  394. def __init__(self, host, options, python_executable=None):
  395. self._host = host
  396. self._path_finder = PathFinder(self._host.filesystem)
  397. self._options = options
  398. self._python_executable = python_executable
  399. self._tasks = contextlib.ExitStack()
  400. self._validate_options()
  401. def _path_from_target(self, *components):
  402. return self._path_finder.path_from_chromium_base('out',
  403. self._options.target,
  404. *components)
  405. def _validate_options(self):
  406. """Validate product-specific command-line options.
  407. The validity of some options may depend on the product. We check these
  408. options here instead of at parse time because the product itself is an
  409. option and the parser cannot handle that dependency.
  410. The test environment will not be set up at this point, so checks should
  411. not depend on external resources.
  412. Raises:
  413. ValueError: When the given options are invalid for this product.
  414. The user will see the error's message (formatted with
  415. `argparse`, not a traceback) and the program will exit early,
  416. which avoids wasted runtime.
  417. """
  418. @contextlib.contextmanager
  419. def test_env(self):
  420. """Set up and clean up the test environment."""
  421. with self._tasks:
  422. yield
  423. @property
  424. def wpt_args(self):
  425. """list[str]: Arguments to add to a 'wpt run' command."""
  426. args = []
  427. version = self.get_version() # pylint: disable=assignment-from-none
  428. if version:
  429. args.append('--browser-version=%s' % version)
  430. webdriver = self.webdriver_binary
  431. if webdriver and self._host.filesystem.exists(webdriver):
  432. args.append('--webdriver-binary=%s' % webdriver)
  433. return args
  434. @property
  435. def metadata_builder_args(self):
  436. """list[str]: Arguments to add to the WPT metadata builder command."""
  437. return ['--additional-expectations=%s' % expectation
  438. for expectation in self.expectations]
  439. @property
  440. def expectations(self):
  441. """list[str]: Paths to additional expectations to build metadata for."""
  442. return list(self._options.additional_expectations)
  443. def get_version(self):
  444. """Get the product version, if available."""
  445. return None
  446. @property
  447. def webdriver_binary(self):
  448. """Optional[str]: Path to the webdriver binary, if available."""
  449. return self._options.webdriver_binary
  450. class Chrome(Product):
  451. name = 'chrome'
  452. @property
  453. def wpt_args(self):
  454. wpt_args = list(super().wpt_args)
  455. wpt_args.extend([
  456. '--binary=%s' % self.binary,
  457. '--processes=%d' % self._options.processes,
  458. ])
  459. return wpt_args
  460. @property
  461. def metadata_builder_args(self):
  462. args = list(super().metadata_builder_args)
  463. path_to_wpt_root = self._path_finder.path_from_web_tests(
  464. self._path_finder.wpt_prefix())
  465. # TODO(crbug/1299650): Strip trailing '/'. Otherwise,
  466. # build_wpt_metadata.py will not build correctly filesystem paths
  467. # correctly.
  468. path_to_wpt_root = self._host.filesystem.normpath(path_to_wpt_root)
  469. args.extend([
  470. '--no-process-baselines',
  471. '--checked-in-metadata-dir=%s' % path_to_wpt_root,
  472. ])
  473. return args
  474. @property
  475. def expectations(self):
  476. expectations = list(super().expectations)
  477. expectations.append(
  478. self._path_finder.path_from_web_tests('WPTOverrideExpectations'))
  479. return expectations
  480. @property
  481. def binary(self):
  482. binary_path = 'chrome'
  483. if self._host.platform.is_win():
  484. binary_path += '.exe'
  485. elif self._host.platform.is_mac():
  486. binary_path = self._host.filesystem.join(
  487. 'Chromium.app',
  488. 'Contents',
  489. 'MacOS',
  490. 'Chromium')
  491. return self._path_from_target(binary_path)
  492. @property
  493. def webdriver_binary(self):
  494. default_binary = 'chromedriver'
  495. if self._host.platform.is_win():
  496. default_binary += '.exe'
  497. return (super().webdriver_binary
  498. or self._path_from_target(default_binary))
  499. class ChromeiOS(Product):
  500. name = 'chrome_ios'
  501. @property
  502. def wpt_args(self):
  503. wpt_args = list(super().wpt_args)
  504. wpt_args.extend([
  505. '--processes=%d' % self._options.processes,
  506. ])
  507. return wpt_args
  508. @property
  509. def expectations(self):
  510. expectations = list(super().expectations)
  511. expectations.append(
  512. self._path_finder.path_from_web_tests('WPTOverrideExpectations'))
  513. return expectations
  514. @contextlib.contextmanager
  515. def _install_apk(device, path):
  516. """Helper context manager for ensuring a device uninstalls an APK."""
  517. device.Install(path)
  518. try:
  519. yield
  520. finally:
  521. device.Uninstall(path)
  522. class ChromeAndroidBase(Product):
  523. def __init__(self, host, options, python_executable=None):
  524. super().__init__(host, options, python_executable)
  525. self.devices = {}
  526. @contextlib.contextmanager
  527. def test_env(self):
  528. with super().test_env():
  529. devil_chromium.Initialize(adb_path=self._options.adb_binary)
  530. if not self._options.adb_binary:
  531. self._options.adb_binary = devil_env.config.FetchPath('adb')
  532. devices = self._tasks.enter_context(get_devices(self._options))
  533. if not devices:
  534. raise Exception(
  535. 'No devices attached to this host. '
  536. "Make sure to provide '--avd-config' "
  537. 'if using only emulators.')
  538. self.provision_devices(devices)
  539. yield
  540. @property
  541. def wpt_args(self):
  542. wpt_args = list(super().wpt_args)
  543. for serial in self.devices:
  544. wpt_args.append('--device-serial=%s' % serial)
  545. package_name = self.get_browser_package_name()
  546. if package_name:
  547. wpt_args.append('--package-name=%s' % package_name)
  548. if self._options.adb_binary:
  549. wpt_args.append('--adb-binary=%s' % self._options.adb_binary)
  550. return wpt_args
  551. @property
  552. def metadata_builder_args(self):
  553. args = list(super().metadata_builder_args)
  554. args.extend([
  555. '--android-product=%s' % self.name,
  556. '--use-subtest-results',
  557. ])
  558. return args
  559. @property
  560. def expectations(self):
  561. expectations = list(super().expectations)
  562. expectations.append(ANDROID_DISABLED_TESTS)
  563. maybe_path = PRODUCTS_TO_EXPECTATION_FILE_PATHS.get(self.name)
  564. if (maybe_path
  565. and not self._options.ignore_browser_specific_expectations):
  566. expectations.append(maybe_path)
  567. return expectations
  568. def get_version(self):
  569. version_provider = self.get_version_provider_package_name()
  570. if self.devices and version_provider:
  571. # Assume devices are identically provisioned, so select any.
  572. device = list(self.devices.values())[0]
  573. try:
  574. version = device.GetApplicationVersion(version_provider)
  575. logger.info('Product version: %s %s (package: %r)',
  576. self.name, version, version_provider)
  577. return version
  578. except CommandFailedError:
  579. logger.warning('Failed to retrieve version of %s (package: %r)',
  580. self.name, version_provider)
  581. return None
  582. @property
  583. def webdriver_binary(self):
  584. default_binary = self._path_from_target('clang_x64',
  585. 'chromedriver')
  586. return super().webdriver_binary or default_binary
  587. def get_browser_package_name(self):
  588. """Get the name of the package to run tests against.
  589. For WebView and WebLayer, this package is the shell.
  590. Returns:
  591. Optional[str]: The name of a package installed on the devices or
  592. `None` to use wpt's best guess of the runnable package.
  593. See Also:
  594. https://github.com/web-platform-tests/wpt/blob/merge_pr_33203/tools/wpt/browser.py#L867-L924
  595. """
  596. if self._options.package_name:
  597. return self._options.package_name
  598. if self._options.browser_apk:
  599. with contextlib.suppress(apk_helper.ApkHelperError):
  600. return apk_helper.GetPackageName(self._options.browser_apk)
  601. return None
  602. def get_version_provider_package_name(self):
  603. """Get the name of the package containing the product version.
  604. Some Android products are made up of multiple packages with decoupled
  605. "versionName" fields. This method identifies the package whose
  606. "versionName" should be consider the product's version.
  607. Returns:
  608. Optional[str]: The name of a package installed on the devices or
  609. `None` to use wpt's best guess of the version.
  610. See Also:
  611. https://github.com/web-platform-tests/wpt/blob/merge_pr_33203/tools/wpt/run.py#L810-L816
  612. https://github.com/web-platform-tests/wpt/blob/merge_pr_33203/tools/wpt/browser.py#L850-L924
  613. """
  614. # Assume the product is a single APK.
  615. return self.get_browser_package_name()
  616. def provision_devices(self, devices):
  617. """Provisions a set of Android devices in parallel."""
  618. contexts = [self._provision_device(device) for device in devices]
  619. self._tasks.enter_context(SyncParallelizer(contexts))
  620. for device in devices:
  621. if device.serial in self.devices:
  622. raise Exception('duplicate device serial: %s' % device.serial)
  623. self.devices[device.serial] = device
  624. self._tasks.callback(self.devices.pop, device.serial, None)
  625. @contextlib.contextmanager
  626. def _provision_device(self, device):
  627. """Provision a single Android device for a test.
  628. This method will be executed in parallel on all devices, so
  629. it is crucial that it is thread safe.
  630. """
  631. with contextlib.ExitStack() as exit_stack:
  632. if self._options.browser_apk:
  633. exit_stack.enter_context(
  634. _install_apk(device, self._options.browser_apk))
  635. for apk in self._options.additional_apk:
  636. exit_stack.enter_context(_install_apk(device, apk))
  637. logger.info('Provisioned device (serial: %s)', device.serial)
  638. yield
  639. @contextlib.contextmanager
  640. def _install_webview_from_release(device, channel, python_executable=None):
  641. script_path = os.path.join(SRC_DIR, 'clank', 'bin', 'install_webview.py')
  642. python_executable = python_executable or sys.executable
  643. command = [python_executable, script_path, '-s', device.serial, '--channel',
  644. channel]
  645. exit_code = common.run_command(command)
  646. if exit_code != 0:
  647. raise Exception('failed to install webview from release '
  648. '(serial: %r, channel: %r, exit code: %d)'
  649. % (device.serial, channel, exit_code))
  650. yield
  651. class WebLayer(ChromeAndroidBase):
  652. name = ANDROID_WEBLAYER
  653. aliases = ['weblayer']
  654. @property
  655. def wpt_args(self):
  656. args = list(super().wpt_args)
  657. args.append('--test-type=testharness')
  658. return args
  659. def get_browser_package_name(self):
  660. return (super().get_browser_package_name()
  661. or 'org.chromium.weblayer.shell')
  662. def get_version_provider_package_name(self):
  663. if self._options.additional_apk:
  664. support_apk = self._options.additional_apk[0]
  665. with contextlib.suppress(apk_helper.ApkHelperError):
  666. return apk_helper.GetPackageName(support_apk)
  667. return super().get_version_provider_package_name()
  668. class WebView(ChromeAndroidBase):
  669. name = ANDROID_WEBVIEW
  670. aliases = ['webview']
  671. def _install_webview(self, device):
  672. # Prioritize local builds.
  673. if self._options.webview_provider:
  674. return webview_app.UseWebViewProvider(
  675. device,
  676. self._options.webview_provider)
  677. assert self._options.release_channel, 'no webview install method'
  678. return _install_webview_from_release(
  679. device,
  680. self._options.release_channel,
  681. self._python_executable)
  682. def _validate_options(self):
  683. super()._validate_options()
  684. if not self._options.webview_provider \
  685. and not self._options.release_channel:
  686. raise ValueError(
  687. "Must provide either '--webview-provider' or "
  688. "'--release-channel' to install WebView.")
  689. def get_browser_package_name(self):
  690. return (super().get_browser_package_name()
  691. or 'org.chromium.webview_shell')
  692. def get_version_provider_package_name(self):
  693. # Prioritize using the webview provider, not the shell, since the
  694. # provider is distributed to end users. The shell is developer-facing,
  695. # so its version is usually not actively updated.
  696. if self._options.webview_provider:
  697. with contextlib.suppress(apk_helper.ApkHelperError):
  698. return apk_helper.GetPackageName(self._options.webview_provider)
  699. return super().get_version_provider_package_name()
  700. @contextlib.contextmanager
  701. def _provision_device(self, device):
  702. with self._install_webview(device), super()._provision_device(device):
  703. yield
  704. class ChromeAndroid(ChromeAndroidBase):
  705. name = CHROME_ANDROID
  706. aliases = ['clank']
  707. def _validate_options(self):
  708. super()._validate_options()
  709. if not self._options.package_name and not self._options.browser_apk:
  710. raise ValueError(
  711. "Must provide either '--package-name' or '--browser-apk' "
  712. 'for %r.' % self.name)
  713. def add_emulator_args(parser):
  714. parser.add_argument(
  715. '--avd-config',
  716. type=os.path.realpath,
  717. help=('Path to the avd config. Required for Android products. '
  718. '(See //tools/android/avd/proto for message definition '
  719. 'and existing *.textpb files.)'))
  720. parser.add_argument(
  721. '--emulator-window',
  722. action='store_true',
  723. default=False,
  724. help='Enable graphical window display on the emulator.')
  725. def _make_product_registry():
  726. """Create a mapping from all product names (including aliases) to their
  727. respective classes.
  728. """
  729. product_registry = {}
  730. product_classes = [Chrome, ChromeiOS]
  731. if _ANDROID_ENABLED:
  732. product_classes.extend([ChromeAndroid, WebView, WebLayer])
  733. for product_cls in product_classes:
  734. names = [product_cls.name] + product_cls.aliases
  735. product_registry.update((name, product_cls) for name in names)
  736. return product_registry
  737. _product_registry = _make_product_registry()
  738. @contextlib.contextmanager
  739. def get_device(args):
  740. with get_devices(args) as devices:
  741. yield None if not devices else devices[0]
  742. @contextlib.contextmanager
  743. def get_devices(args):
  744. if not _ANDROID_ENABLED:
  745. raise Exception('Android is not available')
  746. instances = []
  747. try:
  748. if args.avd_config:
  749. avd_config = avd.AvdConfig(args.avd_config)
  750. logger.warning('Installing emulator from %s', args.avd_config)
  751. avd_config.Install()
  752. for _ in range(max(args.processes, 1)):
  753. instance = avd_config.CreateInstance()
  754. instances.append(instance)
  755. SyncParallelizer(instances).Start(
  756. writable_system=True, window=args.emulator_window)
  757. #TODO(weizhong): when choose device, make sure abi matches with target
  758. yield device_utils.DeviceUtils.HealthyDevices()
  759. finally:
  760. SyncParallelizer(instances).Stop()
  761. def main():
  762. adapter = WPTAdapter()
  763. return adapter.run_test()
  764. # This is not really a "script test" so does not need to manually add
  765. # any additional compile targets.
  766. def main_compile_targets(args):
  767. json.dump([], args.output)
  768. if __name__ == '__main__':
  769. # Conform minimally to the protocol defined by ScriptTest.
  770. if 'compile_targets' in sys.argv:
  771. funcs = {
  772. 'run': None,
  773. 'compile_targets': main_compile_targets,
  774. }
  775. sys.exit(common.run_script(sys.argv[1:], funcs))
  776. sys.exit(main())