run_finch_smoke_tests_android.py 30 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834
  1. #!/usr/bin/env vpython3
  2. # Copyright 2021 The Chromium Authors. All rights reserved.
  3. # Use of this source code is governed by a BSD-style license that can be
  4. # found in the LICENSE file.
  5. import argparse
  6. import contextlib
  7. import json
  8. import logging
  9. import os
  10. import posixpath
  11. import re
  12. import shutil
  13. import sys
  14. import tempfile
  15. import time
  16. from collections import OrderedDict
  17. from PIL import Image
  18. SRC_DIR = os.path.abspath(
  19. os.path.join(os.path.dirname(__file__), os.pardir, os.pardir))
  20. PAR_DIR = os.path.join(SRC_DIR, 'testing')
  21. OUT_DIR = os.path.join(SRC_DIR, 'out', 'Release')
  22. BLINK_DIR = os.path.join(SRC_DIR, 'third_party', 'blink')
  23. BLINK_TOOLS = os.path.join(BLINK_DIR, 'tools')
  24. BLINK_WEB_TESTS = os.path.join(BLINK_DIR, 'web_tests')
  25. BUILD_ANDROID = os.path.join(SRC_DIR, 'build', 'android')
  26. CATAPULT_DIR = os.path.join(SRC_DIR, 'third_party', 'catapult')
  27. PYUTILS = os.path.join(CATAPULT_DIR, 'common', 'py_utils')
  28. # Protocall buffer directories to import
  29. PYPROTO_LIB = os.path.join(OUT_DIR, 'pyproto', 'google')
  30. WEBVIEW_VARIATIONS_PROTO = os.path.join(OUT_DIR, 'pyproto',
  31. 'android_webview', 'proto')
  32. if PYUTILS not in sys.path:
  33. sys.path.append(PYUTILS)
  34. if BUILD_ANDROID not in sys.path:
  35. sys.path.append(BUILD_ANDROID)
  36. if BLINK_TOOLS not in sys.path:
  37. sys.path.append(BLINK_TOOLS)
  38. if PYPROTO_LIB not in sys.path:
  39. sys.path.append(PYPROTO_LIB)
  40. if WEBVIEW_VARIATIONS_PROTO not in sys.path:
  41. sys.path.append(WEBVIEW_VARIATIONS_PROTO)
  42. sys.path.append(PAR_DIR)
  43. if 'compile_targets' not in sys.argv:
  44. import aw_variations_seed_pb2
  45. import devil_chromium
  46. import wpt_common
  47. from blinkpy.web_tests.models import test_failures
  48. from blinkpy.web_tests.port.android import (
  49. ANDROID_WEBLAYER, ANDROID_WEBVIEW, CHROME_ANDROID)
  50. from devil import devil_env
  51. from devil.android import apk_helper
  52. from devil.android import device_temp_file
  53. from devil.android import flag_changer
  54. from devil.android import logcat_monitor
  55. from devil.android.tools import script_common
  56. from devil.android.tools import system_app
  57. from devil.android.tools import webview_app
  58. from devil.utils import logging_common
  59. from pylib.local.device import local_device_environment
  60. from pylib.local.emulator import avd
  61. from py_utils.tempfile_ext import NamedTemporaryDirectory
  62. from scripts import common
  63. from skia_gold_infra.finch_skia_gold_properties import FinchSkiaGoldProperties
  64. from skia_gold_infra import finch_skia_gold_session_manager
  65. from skia_gold_infra import finch_skia_gold_utils
  66. from run_wpt_tests import add_emulator_args, get_device
  67. LOGCAT_TAG = 'finch_test_runner_py'
  68. LOGCAT_FILTERS = [
  69. 'chromium:v',
  70. 'cr_*:v',
  71. 'DEBUG:I',
  72. 'StrictMode:D',
  73. 'WebView*:v',
  74. '%s:I' % LOGCAT_TAG
  75. ]
  76. logger = logging.getLogger(__name__)
  77. logger.setLevel(logging.INFO)
  78. TEST_CASES = {}
  79. def _merge_results_dicts(dict_to_merge, test_results_dict):
  80. if 'actual' in dict_to_merge:
  81. test_results_dict.update(dict_to_merge)
  82. return
  83. for key in dict_to_merge.keys():
  84. _merge_results_dicts(dict_to_merge[key],
  85. test_results_dict.setdefault(key, {}))
  86. # pylint: disable=super-with-arguments
  87. class FinchTestCase(wpt_common.BaseWptScriptAdapter):
  88. def __init__(self, device):
  89. super(FinchTestCase, self).__init__()
  90. self._device = device
  91. self.parse_args()
  92. self._browser_apk_helper = apk_helper.ToHelper(self.options.browser_apk)
  93. self.browser_package_name = self._browser_apk_helper.GetPackageName()
  94. self.browser_activity_name = (self.options.browser_activity_name or
  95. self.default_browser_activity_name)
  96. self.layout_test_results_subdir = None
  97. self.test_specific_browser_args = []
  98. if self.options.webview_provider_apk:
  99. self.webview_provider_package_name = (
  100. apk_helper.GetPackageName(self.options.webview_provider_apk))
  101. # Initialize the Skia Gold session manager
  102. self._skia_gold_corpus = 'finch-smoke-tests'
  103. self._skia_gold_tmp_dir = None
  104. self._skia_gold_session_manager = None
  105. @classmethod
  106. def app_user_sub_dir(cls):
  107. """Returns sub directory within user directory"""
  108. return 'app_%s' % cls.product_name()
  109. @classmethod
  110. def product_name(cls):
  111. raise NotImplementedError
  112. @property
  113. def tests(self):
  114. return [
  115. 'dom/collections/HTMLCollection-delete.html',
  116. 'dom/collections/HTMLCollection-supported-property-names.html',
  117. 'dom/collections/HTMLCollection-supported-property-indices.html',
  118. ]
  119. @property
  120. def pixel_tests(self):
  121. return []
  122. @property
  123. def default_browser_activity_name(self):
  124. raise NotImplementedError
  125. @property
  126. def default_finch_seed_path(self):
  127. raise NotImplementedError
  128. @classmethod
  129. def finch_seed_download_args(cls):
  130. return []
  131. def new_seed_downloaded(self):
  132. # TODO(crbug.com/1285152): Implement seed download test
  133. # for Chrome and WebLayer.
  134. return True
  135. @contextlib.contextmanager
  136. def _archive_logcat(self, filename, endpoint_name):
  137. with logcat_monitor.LogcatMonitor(
  138. self._device.adb,
  139. filter_specs=LOGCAT_FILTERS,
  140. output_file=filename,
  141. check_error=False):
  142. try:
  143. self._device.RunShellCommand(['log', '-p', 'i', '-t', LOGCAT_TAG,
  144. 'START {}'.format(endpoint_name)],
  145. check_return=True)
  146. yield
  147. finally:
  148. self._device.RunShellCommand(['log', '-p', 'i', '-t', LOGCAT_TAG,
  149. 'END {}'.format(endpoint_name)],
  150. check_return=True)
  151. def parse_args(self, args=None):
  152. super(FinchTestCase, self).parse_args(args)
  153. if (not self.options.finch_seed_path or
  154. not os.path.exists(self.options.finch_seed_path)):
  155. self.options.finch_seed_path = self.default_finch_seed_path
  156. def __enter__(self):
  157. self._device.EnableRoot()
  158. # Run below commands to ensure that the device can download a seed
  159. self._device.adb.Emu(['power', 'ac', 'on'])
  160. self._device.RunShellCommand(['svc', 'wifi', 'enable'])
  161. self._skia_gold_tmp_dir = tempfile.mkdtemp()
  162. self._skia_gold_session_manager = (
  163. finch_skia_gold_session_manager.FinchSkiaGoldSessionManager(
  164. self._skia_gold_tmp_dir, FinchSkiaGoldProperties(self.options)))
  165. return self
  166. def __exit__(self, exc_type, exc_val, exc_tb):
  167. self._skia_gold_session_manager = None
  168. if self._skia_gold_tmp_dir:
  169. shutil.rmtree(self._skia_gold_tmp_dir)
  170. self._skia_gold_tmp_dir = None
  171. @property
  172. def rest_args(self):
  173. rest_args = super(FinchTestCase, self).rest_args
  174. rest_args.extend([
  175. '--device-serial',
  176. self._device.serial,
  177. '--webdriver-binary',
  178. os.path.join('clang_x64', 'chromedriver'),
  179. '--symbols-path',
  180. self.output_directory,
  181. '--package-name',
  182. self.browser_package_name,
  183. '--keep-app-data-directory',
  184. '--test-type=testharness',
  185. ])
  186. for binary_arg in self.browser_command_line_args():
  187. rest_args.append('--binary-arg=%s' % binary_arg)
  188. for test in self.tests:
  189. rest_args.extend(['--include', test])
  190. return rest_args
  191. @classmethod
  192. def add_common_arguments(cls, parser):
  193. parser.add_argument('--test-case',
  194. choices=TEST_CASES.keys(),
  195. # TODO(rmhasan): Remove default values after
  196. # adding arguments to test suites. Also make
  197. # this argument required.
  198. default='webview',
  199. help='Name of test case')
  200. parser.add_argument('--finch-seed-path',
  201. type=os.path.realpath,
  202. help='Path to the finch seed')
  203. parser.add_argument('--browser-apk',
  204. '--webview-shell-apk',
  205. '--weblayer-shell-apk',
  206. help='Path to the browser apk',
  207. type=os.path.realpath,
  208. required=True)
  209. parser.add_argument('--webview-provider-apk',
  210. type=os.path.realpath,
  211. help='Path to the WebView provider apk')
  212. parser.add_argument('--additional-apk',
  213. action='append',
  214. type=os.path.realpath,
  215. default=[],
  216. help='List of additional apk\'s to install')
  217. parser.add_argument('--browser-activity-name',
  218. action='store',
  219. help='Browser activity name')
  220. parser.add_argument('--fake-variations-channel',
  221. action='store',
  222. default='stable',
  223. choices=['dev', 'canary', 'beta', 'stable'],
  224. help='Finch seed release channel')
  225. parser.add_argument('-j',
  226. '--processes',
  227. type=lambda processes: max(0, int(processes)),
  228. default=1,
  229. help='Number of emulator to run.')
  230. # Add arguments used by Skia Gold.
  231. FinchSkiaGoldProperties.AddCommandLineArguments(parser)
  232. add_emulator_args(parser)
  233. def add_extra_arguments(self, parser):
  234. super(FinchTestCase, self).add_extra_arguments(parser)
  235. self.add_common_arguments(parser)
  236. def _compare_screenshots_with_baselines(self, all_pixel_tests_results_dict):
  237. """Compare pixel tests screenshots with baselines stored in skia gold
  238. Args:
  239. all_pixel_tests_results_dict: Results dictionary for all pixel tests
  240. Returns:
  241. 1 if there was an error comparing images otherwise 0
  242. """
  243. skia_gold_session = (
  244. self._skia_gold_session_manager.GetSkiaGoldSession(
  245. {'platform': 'android'}, self._skia_gold_corpus))
  246. def _process_test_leaf(test_result_dict):
  247. if ('artifacts' not in test_result_dict or
  248. 'actual_image' not in test_result_dict['artifacts']):
  249. return 0
  250. return_code = 0
  251. artifacts_dict = test_result_dict['artifacts']
  252. curr_artifacts = list(artifacts_dict.keys())
  253. for artifact_name in curr_artifacts:
  254. artifact_path = artifacts_dict[artifact_name][0]
  255. # Compare screenshots to baselines stored in Skia Gold
  256. status, error = skia_gold_session.RunComparison(
  257. artifact_path,
  258. os.path.join(os.path.dirname(self.wpt_output), artifact_path))
  259. if status:
  260. test_result_dict['actual'] = 'FAIL'
  261. all_pixel_tests_results_dict['num_failures_by_type'].setdefault(
  262. 'FAIL', 0)
  263. all_pixel_tests_results_dict['num_failures_by_type']['FAIL'] += 1
  264. triage_link = finch_skia_gold_utils.log_skia_gold_status_code(
  265. skia_gold_session, artifact_path, status, error)
  266. if triage_link:
  267. artifacts_dict['%s_triage_link' % artifact_name] = [triage_link]
  268. return_code = 1
  269. else:
  270. test_result_dict['actual'] = 'PASS'
  271. return return_code
  272. def _process_tests(node):
  273. return_code = 0
  274. if 'actual' in node:
  275. return _process_test_leaf(node)
  276. for next_node in node.values():
  277. return_code |= _process_tests(next_node)
  278. return return_code
  279. return _process_tests(all_pixel_tests_results_dict['tests'])
  280. @contextlib.contextmanager
  281. def install_apks(self):
  282. """Install apks for testing"""
  283. self._device.Uninstall(self.browser_package_name)
  284. self._device.Install(self.options.browser_apk, reinstall=True)
  285. for apk_path in self.options.additional_apk:
  286. self._device.Install(apk_path)
  287. self._device.ClearApplicationState(
  288. self.browser_package_name,
  289. permissions=self._browser_apk_helper.GetPermissions())
  290. # TODO(rmhasan): For R+ test devices, store the files in the
  291. # app's data directory. This is needed for R+ devices because
  292. # of the scoped storage feature.
  293. tests_root_dir = posixpath.join(self._device.GetExternalStoragePath(),
  294. 'chromium_tests_root')
  295. local_device_environment.place_nomedia_on_device(self._device,
  296. tests_root_dir)
  297. # Store screenshot tests on the device's external storage.
  298. for test_file in self.pixel_tests:
  299. self._device.RunShellCommand(
  300. ['mkdir', '-p',
  301. posixpath.join(tests_root_dir,
  302. 'pixel_tests',
  303. posixpath.dirname(test_file))],
  304. check_return=True)
  305. self._device.adb.Push(os.path.join(BLINK_WEB_TESTS, test_file),
  306. posixpath.join(tests_root_dir,
  307. 'pixel_tests',
  308. test_file))
  309. yield
  310. def browser_command_line_args(self):
  311. return (['--fake-variations-channel=%s' %
  312. self.options.fake_variations_channel] +
  313. self.test_specific_browser_args)
  314. def run_tests(self, test_run_variation, all_test_results_dict,
  315. extra_browser_args=None):
  316. """Run browser test on test device
  317. Args:
  318. test_run_variation: Test run variation.
  319. all_test_results_dict: Main results dictionary containing
  320. results for all test variations.
  321. extra_browser_args: Extra browser arguments.
  322. Returns:
  323. True if browser did not crash or False if the browser crashed.
  324. """
  325. isolate_root_dir = os.path.dirname(
  326. self.options.isolated_script_test_output)
  327. logcat_filename = '{}_{}_test_run_logcat.txt'.format(
  328. self.product_name(), test_run_variation)
  329. self.layout_test_results_subdir = ('%s_smoke_test_artifacts' %
  330. test_run_variation)
  331. self.test_specific_browser_args = extra_browser_args or []
  332. with self._archive_logcat(os.path.join(isolate_root_dir, logcat_filename),
  333. '{} {} tests'.format(self.product_name(),
  334. test_run_variation)):
  335. # Make sure the browser is not running before the tests run
  336. self.stop_browser()
  337. ret = super(FinchTestCase, self).run_test()
  338. self.stop_browser()
  339. # Run screen shot tests
  340. pixel_tests_results_dict, pixel_tests_ret = self._run_pixel_tests()
  341. ret |= pixel_tests_ret
  342. self._include_variation_prefix(test_run_variation)
  343. self.process_and_upload_results()
  344. final_logcat_path = os.path.join(isolate_root_dir,
  345. self.layout_test_results_subdir,
  346. logcat_filename)
  347. shutil.move(os.path.join(isolate_root_dir, logcat_filename),
  348. final_logcat_path)
  349. with open(self.wpt_output, 'r') as test_harness_results:
  350. test_harness_results_dict = json.load(test_harness_results)
  351. all_test_results_dict['tests'][test_run_variation] = (
  352. test_harness_results_dict['tests'])
  353. _merge_results_dicts(pixel_tests_results_dict['tests'],
  354. all_test_results_dict['tests'][test_run_variation])
  355. for test_results_dict in (test_harness_results_dict,
  356. pixel_tests_results_dict):
  357. for result, count in test_results_dict['num_failures_by_type'].items():
  358. all_test_results_dict['num_failures_by_type'].setdefault(result, 0)
  359. all_test_results_dict['num_failures_by_type'][result] += count
  360. return ret
  361. def _run_pixel_tests(self):
  362. """Run pixel tests on device
  363. Returns:
  364. A tuple containing a dictionary of pixel test results
  365. and the skia gold status code.
  366. """
  367. tests_root_dir = posixpath.join(
  368. self._device.GetExternalStoragePath(),
  369. 'chromium_tests_root',
  370. 'pixel_tests')
  371. pixel_tests_results_dict = {'tests':{}, 'num_failures_by_type': {}}
  372. for test_file in self.pixel_tests:
  373. try:
  374. # The test result will for each tests will be set after
  375. # comparing the test screenshots to skia gold baselines.
  376. url = 'file://{}'.format(
  377. posixpath.join(tests_root_dir, test_file))
  378. self.start_browser(url)
  379. screenshot_artifact_relpath = os.path.join(
  380. 'pixel_tests_artifacts',
  381. self.layout_test_results_subdir.replace('_artifacts', ''),
  382. self.port.output_filename(test_file,
  383. test_failures.FILENAME_SUFFIX_ACTUAL,
  384. '.png'))
  385. screenshot_artifact_abspath = os.path.join(
  386. os.path.dirname(self.options.isolated_script_test_output),
  387. screenshot_artifact_relpath)
  388. self._device.TakeScreenshot(host_path=screenshot_artifact_abspath)
  389. # Crop away the Android status bar and the WebView shell's support
  390. # action bar. We will do this by removing one fifth of the image
  391. # from the top. We can do this by setting the new top point of the
  392. # image to height / height_factor. height_factor is set to 5.
  393. height_factor = 5
  394. image = Image.open(screenshot_artifact_abspath)
  395. width, height = image.size
  396. cropped_image = image.crop((0, height // height_factor, width, height))
  397. image.close()
  398. cropped_image.save(screenshot_artifact_abspath)
  399. test_results_dict = pixel_tests_results_dict['tests']
  400. for key in test_file.split('/'):
  401. test_results_dict = test_results_dict.setdefault(key, {})
  402. test_results_dict['actual'] = 'PASS'
  403. test_results_dict['expected'] = 'PASS'
  404. test_results_dict['artifacts'] = {
  405. 'actual_image': [screenshot_artifact_relpath]}
  406. finally:
  407. self.stop_browser()
  408. # Compare screenshots with baselines stored in Skia Gold.
  409. return (pixel_tests_results_dict,
  410. self._compare_screenshots_with_baselines(pixel_tests_results_dict))
  411. def _include_variation_prefix(self, test_run_variation):
  412. with open(self.wpt_output, 'r') as test_results_file:
  413. results = json.load(test_results_file)
  414. results.setdefault('metadata', {})['test_name_prefix'] = test_run_variation
  415. with open(self.wpt_output, 'w+') as test_results_file:
  416. json.dump(results, test_results_file)
  417. def stop_browser(self):
  418. logger.info('Stopping package %s', self.browser_package_name)
  419. self._device.ForceStop(self.browser_package_name)
  420. if self.options.webview_provider_apk:
  421. logger.info('Stopping package %s', self.webview_provider_package_name)
  422. self._device.ForceStop(
  423. self.webview_provider_package_name)
  424. def start_browser(self, url=None):
  425. full_activity_name = '%s/%s' % (self.browser_package_name,
  426. self.browser_activity_name)
  427. logger.info('Starting activity %s', full_activity_name)
  428. url = url or 'www.google.com'
  429. self._device.RunShellCommand([
  430. 'am',
  431. 'start',
  432. '-W',
  433. '-n',
  434. full_activity_name,
  435. '-d',
  436. url])
  437. logger.info('Waiting 10 seconds')
  438. time.sleep(10)
  439. def _wait_for_local_state_file(self, local_state_file):
  440. """Wait for local state file to be generated"""
  441. max_wait_time_secs = 120
  442. delta_secs = 10
  443. total_wait_time_secs = 0
  444. self.start_browser()
  445. while total_wait_time_secs < max_wait_time_secs:
  446. if self._device.PathExists(local_state_file):
  447. logger.info('Local state file generated')
  448. self.stop_browser()
  449. return
  450. logger.info('Waiting %d seconds for the local state file to generate',
  451. delta_secs)
  452. time.sleep(delta_secs)
  453. total_wait_time_secs += delta_secs
  454. raise Exception('Timed out waiting for the '
  455. 'local state file to be generated')
  456. def install_seed(self):
  457. """Install finch seed for testing
  458. Returns:
  459. None
  460. """
  461. app_data_dir = posixpath.join(
  462. self._device.GetApplicationDataDirectory(self.browser_package_name),
  463. self.app_user_sub_dir())
  464. device_local_state_file = posixpath.join(app_data_dir, 'Local State')
  465. self._wait_for_local_state_file(device_local_state_file)
  466. with NamedTemporaryDirectory() as tmp_dir:
  467. tmp_ls_path = os.path.join(tmp_dir, 'local_state.json')
  468. self._device.adb.Pull(device_local_state_file, tmp_ls_path)
  469. with open(tmp_ls_path, 'r') as local_state_content, \
  470. open(self.options.finch_seed_path, 'r') as test_seed_content:
  471. local_state_json = json.loads(local_state_content.read())
  472. test_seed_json = json.loads(test_seed_content.read())
  473. # Copy over the seed data and signature
  474. local_state_json['variations_compressed_seed'] = (
  475. test_seed_json['variations_compressed_seed'])
  476. local_state_json['variations_seed_signature'] = (
  477. test_seed_json['variations_seed_signature'])
  478. with open(os.path.join(tmp_dir, 'new_local_state.json'),
  479. 'w') as new_local_state:
  480. new_local_state.write(json.dumps(local_state_json))
  481. self._device.adb.Push(new_local_state.name, device_local_state_file)
  482. user_id = self._device.GetUidForPackage(self.browser_package_name)
  483. logger.info('Setting owner of Local State file to %r', user_id)
  484. self._device.RunShellCommand(
  485. ['chown', user_id, device_local_state_file], as_root=True)
  486. class ChromeFinchTestCase(FinchTestCase):
  487. @classmethod
  488. def product_name(cls):
  489. """Returns name of product being tested"""
  490. return 'chrome'
  491. @property
  492. def default_finch_seed_path(self):
  493. return os.path.join(SRC_DIR, 'testing', 'scripts',
  494. 'variations_smoke_test_data',
  495. 'variations_seed_stable_chrome_android.json')
  496. @classmethod
  497. def wpt_product_name(cls):
  498. return CHROME_ANDROID
  499. @property
  500. def default_browser_activity_name(self):
  501. return 'org.chromium.chrome.browser.ChromeTabbedActivity'
  502. class WebViewFinchTestCase(FinchTestCase):
  503. @classmethod
  504. def product_name(cls):
  505. """Returns name of product being tested"""
  506. return 'webview'
  507. @classmethod
  508. def wpt_product_name(cls):
  509. return ANDROID_WEBVIEW
  510. @property
  511. def pixel_tests(self):
  512. return super(WebViewFinchTestCase, self).pixel_tests + [
  513. 'external/wpt/svg/pservers/reftests/radialgradient-basic-002.svg',
  514. ]
  515. @classmethod
  516. def finch_seed_download_args(cls):
  517. return [
  518. '--finch-seed-expiration-age=0',
  519. '--finch-seed-min-update-period=0',
  520. '--finch-seed-min-download-period=0',
  521. '--finch-seed-ignore-pending-download',
  522. '--finch-seed-no-charging-requirement']
  523. @property
  524. def default_browser_activity_name(self):
  525. return 'org.chromium.webview_shell.WebViewBrowserActivity'
  526. @property
  527. def default_finch_seed_path(self):
  528. return os.path.join(SRC_DIR, 'testing', 'scripts',
  529. 'variations_smoke_test_data',
  530. 'webview_test_seed')
  531. def new_seed_downloaded(self):
  532. """Checks if a new seed was downloaded
  533. Returns:
  534. True if a new seed was downloaded, otherwise False
  535. """
  536. app_data_dir = posixpath.join(
  537. self._device.GetApplicationDataDirectory(self.browser_package_name),
  538. self.app_user_sub_dir())
  539. remote_seed_path = posixpath.join(app_data_dir, 'variations_seed')
  540. with NamedTemporaryDirectory() as tmp_dir:
  541. current_seed_path = os.path.join(tmp_dir, 'current_seed')
  542. self._device.adb.Pull(remote_seed_path, current_seed_path)
  543. with open(current_seed_path, 'rb') as current_seed_obj, \
  544. open(self.options.finch_seed_path, 'rb') as baseline_seed_obj:
  545. current_seed_content = current_seed_obj.read()
  546. baseline_seed_content = baseline_seed_obj.read()
  547. current_seed = aw_variations_seed_pb2.AwVariationsSeed.FromString(
  548. current_seed_content)
  549. baseline_seed = aw_variations_seed_pb2.AwVariationsSeed.FromString(
  550. baseline_seed_content)
  551. shutil.copy(current_seed_path, os.path.join(OUT_DIR, 'final_seed'))
  552. logger.info("Downloaded seed's signature: %s", current_seed.signature)
  553. logger.info("Baseline seed's signature: %s", baseline_seed.signature)
  554. return current_seed_content != baseline_seed_content
  555. def browser_command_line_args(self):
  556. return (super(WebViewFinchTestCase, self).browser_command_line_args() +
  557. ['--webview-verbose-logging'])
  558. @contextlib.contextmanager
  559. def install_apks(self):
  560. """Install apks for testing"""
  561. with super(WebViewFinchTestCase, self).install_apks(), \
  562. webview_app.UseWebViewProvider(self._device,
  563. self.options.webview_provider_apk):
  564. yield
  565. def install_seed(self):
  566. """Install finch seed for testing
  567. Returns:
  568. None
  569. """
  570. logcat_file = os.path.join(
  571. os.path.dirname(self.options.isolated_script_test_output),
  572. 'install_seed_for_on_device.txt')
  573. with self._archive_logcat(
  574. logcat_file,
  575. 'install seed on device {}'.format(self._device.serial)):
  576. app_data_dir = posixpath.join(
  577. self._device.GetApplicationDataDirectory(self.browser_package_name),
  578. self.app_user_sub_dir())
  579. self._device.RunShellCommand(['mkdir', '-p', app_data_dir],
  580. run_as=self.browser_package_name)
  581. seed_path = posixpath.join(app_data_dir, 'variations_seed')
  582. seed_new_path = posixpath.join(app_data_dir, 'variations_seed_new')
  583. seed_stamp = posixpath.join(app_data_dir, 'variations_stamp')
  584. self._device.adb.Push(self.options.finch_seed_path, seed_path)
  585. self._device.adb.Push(self.options.finch_seed_path, seed_new_path)
  586. self._device.RunShellCommand(
  587. ['touch', seed_stamp], check_return=True,
  588. run_as=self.browser_package_name)
  589. # We need to make the WebView shell package an owner of the seeds,
  590. # see crbug.com/1191169#c19
  591. user_id = self._device.GetUidForPackage(self.browser_package_name)
  592. logger.info('Setting owner of seed files to %r', user_id)
  593. self._device.RunShellCommand(['chown', user_id, seed_path], as_root=True)
  594. self._device.RunShellCommand(
  595. ['chown', user_id, seed_new_path], as_root=True)
  596. class WebLayerFinchTestCase(FinchTestCase):
  597. @classmethod
  598. def product_name(cls):
  599. """Returns name of product being tested"""
  600. return 'weblayer'
  601. @classmethod
  602. def wpt_product_name(cls):
  603. return ANDROID_WEBLAYER
  604. @property
  605. def default_browser_activity_name(self):
  606. return 'org.chromium.weblayer.shell.WebLayerShellActivity'
  607. @property
  608. def default_finch_seed_path(self):
  609. return os.path.join(SRC_DIR, 'testing', 'scripts',
  610. 'variations_smoke_test_data',
  611. 'variations_seed_stable_weblayer.json')
  612. @contextlib.contextmanager
  613. def install_apks(self):
  614. """Install apks for testing"""
  615. with super(WebLayerFinchTestCase, self).install_apks(), \
  616. webview_app.UseWebViewProvider(self._device,
  617. self.options.webview_provider_apk):
  618. yield
  619. def main(args):
  620. TEST_CASES.update(
  621. {p.product_name(): p
  622. for p in [ChromeFinchTestCase, WebViewFinchTestCase,
  623. WebLayerFinchTestCase]})
  624. # Unfortunately, there's a circular dependency between the parser made
  625. # available from `FinchTestCase.add_extra_arguments` and the selection of the
  626. # correct test case. The workaround is a second parser used in `main` only
  627. # that shares some arguments with the script adapter parser. The second parser
  628. # handles --help, so not all arguments are documented. Important arguments
  629. # added by the script adapter are re-added here for visibility.
  630. parser = argparse.ArgumentParser()
  631. FinchTestCase.add_common_arguments(parser)
  632. parser.add_argument(
  633. '--isolated-script-test-output', type=str,
  634. required=False,
  635. help='path to write test results JSON object to')
  636. script_common.AddDeviceArguments(parser)
  637. script_common.AddEnvironmentArguments(parser)
  638. logging_common.AddLoggingArguments(parser)
  639. options, _ = parser.parse_known_args(args)
  640. with get_device(options) as device, \
  641. TEST_CASES[options.test_case](device) as test_case, \
  642. test_case.install_apks():
  643. devil_chromium.Initialize(adb_path=options.adb_path)
  644. logging_common.InitializeLogging(options)
  645. # TODO(rmhasan): Best practice in Chromium is to allow users to provide
  646. # their own adb binary to avoid adb server restarts. We should add a new
  647. # command line argument to wptrunner so that users can pass the path to
  648. # their adb binary.
  649. platform_tools_path = os.path.dirname(devil_env.config.FetchPath('adb'))
  650. os.environ['PATH'] = os.pathsep.join([platform_tools_path] +
  651. os.environ['PATH'].split(os.pathsep))
  652. test_results_dict = OrderedDict({'version': 3, 'interrupted': False,
  653. 'num_failures_by_type': {}, 'tests': {}})
  654. if test_case.product_name() == 'webview':
  655. ret = test_case.run_tests('without_finch_seed', test_results_dict)
  656. test_case.install_seed()
  657. ret |= test_case.run_tests('with_finch_seed', test_results_dict)
  658. # WebView needs several restarts to fetch and load a new finch seed
  659. # TODO(b/187185389): Figure out why the first restart is needed
  660. ret |= test_case.run_tests('extra_restart', test_results_dict,
  661. test_case.finch_seed_download_args())
  662. # Restart webview+shell to fetch new seed to variations_seed_new
  663. ret |= test_case.run_tests('fetch_new_seed_restart', test_results_dict,
  664. test_case.finch_seed_download_args())
  665. # Restart webview+shell to copy from
  666. # variations_seed_new to variations_seed
  667. ret |= test_case.run_tests('load_new_seed_restart', test_results_dict,
  668. test_case.finch_seed_download_args())
  669. else:
  670. test_case.install_seed()
  671. ret = test_case.run_tests('with_finch_seed', test_results_dict)
  672. # Clears out the finch seed. Need to run finch_seed tests first.
  673. # See crbug/1305430
  674. device.ClearApplicationState(test_case.browser_package_name)
  675. ret |= test_case.run_tests('without_finch_seed', test_results_dict)
  676. test_results_dict['seconds_since_epoch'] = int(time.time())
  677. test_results_dict['path_delimiter'] = '/'
  678. with open(test_case.options.isolated_script_test_output, 'w') as json_out:
  679. json_out.write(json.dumps(test_results_dict, indent=4))
  680. if not test_case.new_seed_downloaded():
  681. raise Exception('A new seed was not downloaded')
  682. # Return zero exit code if tests pass
  683. return ret
  684. def main_compile_targets(args):
  685. json.dump([], args.output)
  686. if __name__ == '__main__':
  687. if 'compile_targets' in sys.argv:
  688. funcs = {
  689. 'run': None,
  690. 'compile_targets': main_compile_targets,
  691. }
  692. sys.exit(common.run_script(sys.argv[1:], funcs))
  693. sys.exit(main(sys.argv[1:]))