123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834 |
- #!/usr/bin/env vpython3
- # Copyright 2021 The Chromium Authors. All rights reserved.
- # Use of this source code is governed by a BSD-style license that can be
- # found in the LICENSE file.
- import argparse
- import contextlib
- import json
- import logging
- import os
- import posixpath
- import re
- import shutil
- import sys
- import tempfile
- import time
- from collections import OrderedDict
- from PIL import Image
- SRC_DIR = os.path.abspath(
- os.path.join(os.path.dirname(__file__), os.pardir, os.pardir))
- PAR_DIR = os.path.join(SRC_DIR, 'testing')
- OUT_DIR = os.path.join(SRC_DIR, 'out', 'Release')
- BLINK_DIR = os.path.join(SRC_DIR, 'third_party', 'blink')
- BLINK_TOOLS = os.path.join(BLINK_DIR, 'tools')
- BLINK_WEB_TESTS = os.path.join(BLINK_DIR, 'web_tests')
- BUILD_ANDROID = os.path.join(SRC_DIR, 'build', 'android')
- CATAPULT_DIR = os.path.join(SRC_DIR, 'third_party', 'catapult')
- PYUTILS = os.path.join(CATAPULT_DIR, 'common', 'py_utils')
- # Protocall buffer directories to import
- PYPROTO_LIB = os.path.join(OUT_DIR, 'pyproto', 'google')
- WEBVIEW_VARIATIONS_PROTO = os.path.join(OUT_DIR, 'pyproto',
- 'android_webview', 'proto')
- if PYUTILS not in sys.path:
- sys.path.append(PYUTILS)
- if BUILD_ANDROID not in sys.path:
- sys.path.append(BUILD_ANDROID)
- if BLINK_TOOLS not in sys.path:
- sys.path.append(BLINK_TOOLS)
- if PYPROTO_LIB not in sys.path:
- sys.path.append(PYPROTO_LIB)
- if WEBVIEW_VARIATIONS_PROTO not in sys.path:
- sys.path.append(WEBVIEW_VARIATIONS_PROTO)
- sys.path.append(PAR_DIR)
- if 'compile_targets' not in sys.argv:
- import aw_variations_seed_pb2
- import devil_chromium
- import wpt_common
- from blinkpy.web_tests.models import test_failures
- from blinkpy.web_tests.port.android import (
- ANDROID_WEBLAYER, ANDROID_WEBVIEW, CHROME_ANDROID)
- from devil import devil_env
- from devil.android import apk_helper
- from devil.android import device_temp_file
- from devil.android import flag_changer
- from devil.android import logcat_monitor
- from devil.android.tools import script_common
- from devil.android.tools import system_app
- from devil.android.tools import webview_app
- from devil.utils import logging_common
- from pylib.local.device import local_device_environment
- from pylib.local.emulator import avd
- from py_utils.tempfile_ext import NamedTemporaryDirectory
- from scripts import common
- from skia_gold_infra.finch_skia_gold_properties import FinchSkiaGoldProperties
- from skia_gold_infra import finch_skia_gold_session_manager
- from skia_gold_infra import finch_skia_gold_utils
- from run_wpt_tests import add_emulator_args, get_device
- LOGCAT_TAG = 'finch_test_runner_py'
- LOGCAT_FILTERS = [
- 'chromium:v',
- 'cr_*:v',
- 'DEBUG:I',
- 'StrictMode:D',
- 'WebView*:v',
- '%s:I' % LOGCAT_TAG
- ]
- logger = logging.getLogger(__name__)
- logger.setLevel(logging.INFO)
- TEST_CASES = {}
- def _merge_results_dicts(dict_to_merge, test_results_dict):
- if 'actual' in dict_to_merge:
- test_results_dict.update(dict_to_merge)
- return
- for key in dict_to_merge.keys():
- _merge_results_dicts(dict_to_merge[key],
- test_results_dict.setdefault(key, {}))
- # pylint: disable=super-with-arguments
- class FinchTestCase(wpt_common.BaseWptScriptAdapter):
- def __init__(self, device):
- super(FinchTestCase, self).__init__()
- self._device = device
- self.parse_args()
- self._browser_apk_helper = apk_helper.ToHelper(self.options.browser_apk)
- self.browser_package_name = self._browser_apk_helper.GetPackageName()
- self.browser_activity_name = (self.options.browser_activity_name or
- self.default_browser_activity_name)
- self.layout_test_results_subdir = None
- self.test_specific_browser_args = []
- if self.options.webview_provider_apk:
- self.webview_provider_package_name = (
- apk_helper.GetPackageName(self.options.webview_provider_apk))
- # Initialize the Skia Gold session manager
- self._skia_gold_corpus = 'finch-smoke-tests'
- self._skia_gold_tmp_dir = None
- self._skia_gold_session_manager = None
- @classmethod
- def app_user_sub_dir(cls):
- """Returns sub directory within user directory"""
- return 'app_%s' % cls.product_name()
- @classmethod
- def product_name(cls):
- raise NotImplementedError
- @property
- def tests(self):
- return [
- 'dom/collections/HTMLCollection-delete.html',
- 'dom/collections/HTMLCollection-supported-property-names.html',
- 'dom/collections/HTMLCollection-supported-property-indices.html',
- ]
- @property
- def pixel_tests(self):
- return []
- @property
- def default_browser_activity_name(self):
- raise NotImplementedError
- @property
- def default_finch_seed_path(self):
- raise NotImplementedError
- @classmethod
- def finch_seed_download_args(cls):
- return []
- def new_seed_downloaded(self):
- # TODO(crbug.com/1285152): Implement seed download test
- # for Chrome and WebLayer.
- return True
- @contextlib.contextmanager
- def _archive_logcat(self, filename, endpoint_name):
- with logcat_monitor.LogcatMonitor(
- self._device.adb,
- filter_specs=LOGCAT_FILTERS,
- output_file=filename,
- check_error=False):
- try:
- self._device.RunShellCommand(['log', '-p', 'i', '-t', LOGCAT_TAG,
- 'START {}'.format(endpoint_name)],
- check_return=True)
- yield
- finally:
- self._device.RunShellCommand(['log', '-p', 'i', '-t', LOGCAT_TAG,
- 'END {}'.format(endpoint_name)],
- check_return=True)
- def parse_args(self, args=None):
- super(FinchTestCase, self).parse_args(args)
- if (not self.options.finch_seed_path or
- not os.path.exists(self.options.finch_seed_path)):
- self.options.finch_seed_path = self.default_finch_seed_path
- def __enter__(self):
- self._device.EnableRoot()
- # Run below commands to ensure that the device can download a seed
- self._device.adb.Emu(['power', 'ac', 'on'])
- self._device.RunShellCommand(['svc', 'wifi', 'enable'])
- self._skia_gold_tmp_dir = tempfile.mkdtemp()
- self._skia_gold_session_manager = (
- finch_skia_gold_session_manager.FinchSkiaGoldSessionManager(
- self._skia_gold_tmp_dir, FinchSkiaGoldProperties(self.options)))
- return self
- def __exit__(self, exc_type, exc_val, exc_tb):
- self._skia_gold_session_manager = None
- if self._skia_gold_tmp_dir:
- shutil.rmtree(self._skia_gold_tmp_dir)
- self._skia_gold_tmp_dir = None
- @property
- def rest_args(self):
- rest_args = super(FinchTestCase, self).rest_args
- rest_args.extend([
- '--device-serial',
- self._device.serial,
- '--webdriver-binary',
- os.path.join('clang_x64', 'chromedriver'),
- '--symbols-path',
- self.output_directory,
- '--package-name',
- self.browser_package_name,
- '--keep-app-data-directory',
- '--test-type=testharness',
- ])
- for binary_arg in self.browser_command_line_args():
- rest_args.append('--binary-arg=%s' % binary_arg)
- for test in self.tests:
- rest_args.extend(['--include', test])
- return rest_args
- @classmethod
- def add_common_arguments(cls, parser):
- parser.add_argument('--test-case',
- choices=TEST_CASES.keys(),
- # TODO(rmhasan): Remove default values after
- # adding arguments to test suites. Also make
- # this argument required.
- default='webview',
- help='Name of test case')
- parser.add_argument('--finch-seed-path',
- type=os.path.realpath,
- help='Path to the finch seed')
- parser.add_argument('--browser-apk',
- '--webview-shell-apk',
- '--weblayer-shell-apk',
- help='Path to the browser apk',
- type=os.path.realpath,
- required=True)
- parser.add_argument('--webview-provider-apk',
- type=os.path.realpath,
- help='Path to the WebView provider apk')
- parser.add_argument('--additional-apk',
- action='append',
- type=os.path.realpath,
- default=[],
- help='List of additional apk\'s to install')
- parser.add_argument('--browser-activity-name',
- action='store',
- help='Browser activity name')
- parser.add_argument('--fake-variations-channel',
- action='store',
- default='stable',
- choices=['dev', 'canary', 'beta', 'stable'],
- help='Finch seed release channel')
- parser.add_argument('-j',
- '--processes',
- type=lambda processes: max(0, int(processes)),
- default=1,
- help='Number of emulator to run.')
- # Add arguments used by Skia Gold.
- FinchSkiaGoldProperties.AddCommandLineArguments(parser)
- add_emulator_args(parser)
- def add_extra_arguments(self, parser):
- super(FinchTestCase, self).add_extra_arguments(parser)
- self.add_common_arguments(parser)
- def _compare_screenshots_with_baselines(self, all_pixel_tests_results_dict):
- """Compare pixel tests screenshots with baselines stored in skia gold
- Args:
- all_pixel_tests_results_dict: Results dictionary for all pixel tests
- Returns:
- 1 if there was an error comparing images otherwise 0
- """
- skia_gold_session = (
- self._skia_gold_session_manager.GetSkiaGoldSession(
- {'platform': 'android'}, self._skia_gold_corpus))
- def _process_test_leaf(test_result_dict):
- if ('artifacts' not in test_result_dict or
- 'actual_image' not in test_result_dict['artifacts']):
- return 0
- return_code = 0
- artifacts_dict = test_result_dict['artifacts']
- curr_artifacts = list(artifacts_dict.keys())
- for artifact_name in curr_artifacts:
- artifact_path = artifacts_dict[artifact_name][0]
- # Compare screenshots to baselines stored in Skia Gold
- status, error = skia_gold_session.RunComparison(
- artifact_path,
- os.path.join(os.path.dirname(self.wpt_output), artifact_path))
- if status:
- test_result_dict['actual'] = 'FAIL'
- all_pixel_tests_results_dict['num_failures_by_type'].setdefault(
- 'FAIL', 0)
- all_pixel_tests_results_dict['num_failures_by_type']['FAIL'] += 1
- triage_link = finch_skia_gold_utils.log_skia_gold_status_code(
- skia_gold_session, artifact_path, status, error)
- if triage_link:
- artifacts_dict['%s_triage_link' % artifact_name] = [triage_link]
- return_code = 1
- else:
- test_result_dict['actual'] = 'PASS'
- return return_code
- def _process_tests(node):
- return_code = 0
- if 'actual' in node:
- return _process_test_leaf(node)
- for next_node in node.values():
- return_code |= _process_tests(next_node)
- return return_code
- return _process_tests(all_pixel_tests_results_dict['tests'])
- @contextlib.contextmanager
- def install_apks(self):
- """Install apks for testing"""
- self._device.Uninstall(self.browser_package_name)
- self._device.Install(self.options.browser_apk, reinstall=True)
- for apk_path in self.options.additional_apk:
- self._device.Install(apk_path)
- self._device.ClearApplicationState(
- self.browser_package_name,
- permissions=self._browser_apk_helper.GetPermissions())
- # TODO(rmhasan): For R+ test devices, store the files in the
- # app's data directory. This is needed for R+ devices because
- # of the scoped storage feature.
- tests_root_dir = posixpath.join(self._device.GetExternalStoragePath(),
- 'chromium_tests_root')
- local_device_environment.place_nomedia_on_device(self._device,
- tests_root_dir)
- # Store screenshot tests on the device's external storage.
- for test_file in self.pixel_tests:
- self._device.RunShellCommand(
- ['mkdir', '-p',
- posixpath.join(tests_root_dir,
- 'pixel_tests',
- posixpath.dirname(test_file))],
- check_return=True)
- self._device.adb.Push(os.path.join(BLINK_WEB_TESTS, test_file),
- posixpath.join(tests_root_dir,
- 'pixel_tests',
- test_file))
- yield
- def browser_command_line_args(self):
- return (['--fake-variations-channel=%s' %
- self.options.fake_variations_channel] +
- self.test_specific_browser_args)
- def run_tests(self, test_run_variation, all_test_results_dict,
- extra_browser_args=None):
- """Run browser test on test device
- Args:
- test_run_variation: Test run variation.
- all_test_results_dict: Main results dictionary containing
- results for all test variations.
- extra_browser_args: Extra browser arguments.
- Returns:
- True if browser did not crash or False if the browser crashed.
- """
- isolate_root_dir = os.path.dirname(
- self.options.isolated_script_test_output)
- logcat_filename = '{}_{}_test_run_logcat.txt'.format(
- self.product_name(), test_run_variation)
- self.layout_test_results_subdir = ('%s_smoke_test_artifacts' %
- test_run_variation)
- self.test_specific_browser_args = extra_browser_args or []
- with self._archive_logcat(os.path.join(isolate_root_dir, logcat_filename),
- '{} {} tests'.format(self.product_name(),
- test_run_variation)):
- # Make sure the browser is not running before the tests run
- self.stop_browser()
- ret = super(FinchTestCase, self).run_test()
- self.stop_browser()
- # Run screen shot tests
- pixel_tests_results_dict, pixel_tests_ret = self._run_pixel_tests()
- ret |= pixel_tests_ret
- self._include_variation_prefix(test_run_variation)
- self.process_and_upload_results()
- final_logcat_path = os.path.join(isolate_root_dir,
- self.layout_test_results_subdir,
- logcat_filename)
- shutil.move(os.path.join(isolate_root_dir, logcat_filename),
- final_logcat_path)
- with open(self.wpt_output, 'r') as test_harness_results:
- test_harness_results_dict = json.load(test_harness_results)
- all_test_results_dict['tests'][test_run_variation] = (
- test_harness_results_dict['tests'])
- _merge_results_dicts(pixel_tests_results_dict['tests'],
- all_test_results_dict['tests'][test_run_variation])
- for test_results_dict in (test_harness_results_dict,
- pixel_tests_results_dict):
- for result, count in test_results_dict['num_failures_by_type'].items():
- all_test_results_dict['num_failures_by_type'].setdefault(result, 0)
- all_test_results_dict['num_failures_by_type'][result] += count
- return ret
- def _run_pixel_tests(self):
- """Run pixel tests on device
- Returns:
- A tuple containing a dictionary of pixel test results
- and the skia gold status code.
- """
- tests_root_dir = posixpath.join(
- self._device.GetExternalStoragePath(),
- 'chromium_tests_root',
- 'pixel_tests')
- pixel_tests_results_dict = {'tests':{}, 'num_failures_by_type': {}}
- for test_file in self.pixel_tests:
- try:
- # The test result will for each tests will be set after
- # comparing the test screenshots to skia gold baselines.
- url = 'file://{}'.format(
- posixpath.join(tests_root_dir, test_file))
- self.start_browser(url)
- screenshot_artifact_relpath = os.path.join(
- 'pixel_tests_artifacts',
- self.layout_test_results_subdir.replace('_artifacts', ''),
- self.port.output_filename(test_file,
- test_failures.FILENAME_SUFFIX_ACTUAL,
- '.png'))
- screenshot_artifact_abspath = os.path.join(
- os.path.dirname(self.options.isolated_script_test_output),
- screenshot_artifact_relpath)
- self._device.TakeScreenshot(host_path=screenshot_artifact_abspath)
- # Crop away the Android status bar and the WebView shell's support
- # action bar. We will do this by removing one fifth of the image
- # from the top. We can do this by setting the new top point of the
- # image to height / height_factor. height_factor is set to 5.
- height_factor = 5
- image = Image.open(screenshot_artifact_abspath)
- width, height = image.size
- cropped_image = image.crop((0, height // height_factor, width, height))
- image.close()
- cropped_image.save(screenshot_artifact_abspath)
- test_results_dict = pixel_tests_results_dict['tests']
- for key in test_file.split('/'):
- test_results_dict = test_results_dict.setdefault(key, {})
- test_results_dict['actual'] = 'PASS'
- test_results_dict['expected'] = 'PASS'
- test_results_dict['artifacts'] = {
- 'actual_image': [screenshot_artifact_relpath]}
- finally:
- self.stop_browser()
- # Compare screenshots with baselines stored in Skia Gold.
- return (pixel_tests_results_dict,
- self._compare_screenshots_with_baselines(pixel_tests_results_dict))
- def _include_variation_prefix(self, test_run_variation):
- with open(self.wpt_output, 'r') as test_results_file:
- results = json.load(test_results_file)
- results.setdefault('metadata', {})['test_name_prefix'] = test_run_variation
- with open(self.wpt_output, 'w+') as test_results_file:
- json.dump(results, test_results_file)
- def stop_browser(self):
- logger.info('Stopping package %s', self.browser_package_name)
- self._device.ForceStop(self.browser_package_name)
- if self.options.webview_provider_apk:
- logger.info('Stopping package %s', self.webview_provider_package_name)
- self._device.ForceStop(
- self.webview_provider_package_name)
- def start_browser(self, url=None):
- full_activity_name = '%s/%s' % (self.browser_package_name,
- self.browser_activity_name)
- logger.info('Starting activity %s', full_activity_name)
- url = url or 'www.google.com'
- self._device.RunShellCommand([
- 'am',
- 'start',
- '-W',
- '-n',
- full_activity_name,
- '-d',
- url])
- logger.info('Waiting 10 seconds')
- time.sleep(10)
- def _wait_for_local_state_file(self, local_state_file):
- """Wait for local state file to be generated"""
- max_wait_time_secs = 120
- delta_secs = 10
- total_wait_time_secs = 0
- self.start_browser()
- while total_wait_time_secs < max_wait_time_secs:
- if self._device.PathExists(local_state_file):
- logger.info('Local state file generated')
- self.stop_browser()
- return
- logger.info('Waiting %d seconds for the local state file to generate',
- delta_secs)
- time.sleep(delta_secs)
- total_wait_time_secs += delta_secs
- raise Exception('Timed out waiting for the '
- 'local state file to be generated')
- def install_seed(self):
- """Install finch seed for testing
- Returns:
- None
- """
- app_data_dir = posixpath.join(
- self._device.GetApplicationDataDirectory(self.browser_package_name),
- self.app_user_sub_dir())
- device_local_state_file = posixpath.join(app_data_dir, 'Local State')
- self._wait_for_local_state_file(device_local_state_file)
- with NamedTemporaryDirectory() as tmp_dir:
- tmp_ls_path = os.path.join(tmp_dir, 'local_state.json')
- self._device.adb.Pull(device_local_state_file, tmp_ls_path)
- with open(tmp_ls_path, 'r') as local_state_content, \
- open(self.options.finch_seed_path, 'r') as test_seed_content:
- local_state_json = json.loads(local_state_content.read())
- test_seed_json = json.loads(test_seed_content.read())
- # Copy over the seed data and signature
- local_state_json['variations_compressed_seed'] = (
- test_seed_json['variations_compressed_seed'])
- local_state_json['variations_seed_signature'] = (
- test_seed_json['variations_seed_signature'])
- with open(os.path.join(tmp_dir, 'new_local_state.json'),
- 'w') as new_local_state:
- new_local_state.write(json.dumps(local_state_json))
- self._device.adb.Push(new_local_state.name, device_local_state_file)
- user_id = self._device.GetUidForPackage(self.browser_package_name)
- logger.info('Setting owner of Local State file to %r', user_id)
- self._device.RunShellCommand(
- ['chown', user_id, device_local_state_file], as_root=True)
- class ChromeFinchTestCase(FinchTestCase):
- @classmethod
- def product_name(cls):
- """Returns name of product being tested"""
- return 'chrome'
- @property
- def default_finch_seed_path(self):
- return os.path.join(SRC_DIR, 'testing', 'scripts',
- 'variations_smoke_test_data',
- 'variations_seed_stable_chrome_android.json')
- @classmethod
- def wpt_product_name(cls):
- return CHROME_ANDROID
- @property
- def default_browser_activity_name(self):
- return 'org.chromium.chrome.browser.ChromeTabbedActivity'
- class WebViewFinchTestCase(FinchTestCase):
- @classmethod
- def product_name(cls):
- """Returns name of product being tested"""
- return 'webview'
- @classmethod
- def wpt_product_name(cls):
- return ANDROID_WEBVIEW
- @property
- def pixel_tests(self):
- return super(WebViewFinchTestCase, self).pixel_tests + [
- 'external/wpt/svg/pservers/reftests/radialgradient-basic-002.svg',
- ]
- @classmethod
- def finch_seed_download_args(cls):
- return [
- '--finch-seed-expiration-age=0',
- '--finch-seed-min-update-period=0',
- '--finch-seed-min-download-period=0',
- '--finch-seed-ignore-pending-download',
- '--finch-seed-no-charging-requirement']
- @property
- def default_browser_activity_name(self):
- return 'org.chromium.webview_shell.WebViewBrowserActivity'
- @property
- def default_finch_seed_path(self):
- return os.path.join(SRC_DIR, 'testing', 'scripts',
- 'variations_smoke_test_data',
- 'webview_test_seed')
- def new_seed_downloaded(self):
- """Checks if a new seed was downloaded
- Returns:
- True if a new seed was downloaded, otherwise False
- """
- app_data_dir = posixpath.join(
- self._device.GetApplicationDataDirectory(self.browser_package_name),
- self.app_user_sub_dir())
- remote_seed_path = posixpath.join(app_data_dir, 'variations_seed')
- with NamedTemporaryDirectory() as tmp_dir:
- current_seed_path = os.path.join(tmp_dir, 'current_seed')
- self._device.adb.Pull(remote_seed_path, current_seed_path)
- with open(current_seed_path, 'rb') as current_seed_obj, \
- open(self.options.finch_seed_path, 'rb') as baseline_seed_obj:
- current_seed_content = current_seed_obj.read()
- baseline_seed_content = baseline_seed_obj.read()
- current_seed = aw_variations_seed_pb2.AwVariationsSeed.FromString(
- current_seed_content)
- baseline_seed = aw_variations_seed_pb2.AwVariationsSeed.FromString(
- baseline_seed_content)
- shutil.copy(current_seed_path, os.path.join(OUT_DIR, 'final_seed'))
- logger.info("Downloaded seed's signature: %s", current_seed.signature)
- logger.info("Baseline seed's signature: %s", baseline_seed.signature)
- return current_seed_content != baseline_seed_content
- def browser_command_line_args(self):
- return (super(WebViewFinchTestCase, self).browser_command_line_args() +
- ['--webview-verbose-logging'])
- @contextlib.contextmanager
- def install_apks(self):
- """Install apks for testing"""
- with super(WebViewFinchTestCase, self).install_apks(), \
- webview_app.UseWebViewProvider(self._device,
- self.options.webview_provider_apk):
- yield
- def install_seed(self):
- """Install finch seed for testing
- Returns:
- None
- """
- logcat_file = os.path.join(
- os.path.dirname(self.options.isolated_script_test_output),
- 'install_seed_for_on_device.txt')
- with self._archive_logcat(
- logcat_file,
- 'install seed on device {}'.format(self._device.serial)):
- app_data_dir = posixpath.join(
- self._device.GetApplicationDataDirectory(self.browser_package_name),
- self.app_user_sub_dir())
- self._device.RunShellCommand(['mkdir', '-p', app_data_dir],
- run_as=self.browser_package_name)
- seed_path = posixpath.join(app_data_dir, 'variations_seed')
- seed_new_path = posixpath.join(app_data_dir, 'variations_seed_new')
- seed_stamp = posixpath.join(app_data_dir, 'variations_stamp')
- self._device.adb.Push(self.options.finch_seed_path, seed_path)
- self._device.adb.Push(self.options.finch_seed_path, seed_new_path)
- self._device.RunShellCommand(
- ['touch', seed_stamp], check_return=True,
- run_as=self.browser_package_name)
- # We need to make the WebView shell package an owner of the seeds,
- # see crbug.com/1191169#c19
- user_id = self._device.GetUidForPackage(self.browser_package_name)
- logger.info('Setting owner of seed files to %r', user_id)
- self._device.RunShellCommand(['chown', user_id, seed_path], as_root=True)
- self._device.RunShellCommand(
- ['chown', user_id, seed_new_path], as_root=True)
- class WebLayerFinchTestCase(FinchTestCase):
- @classmethod
- def product_name(cls):
- """Returns name of product being tested"""
- return 'weblayer'
- @classmethod
- def wpt_product_name(cls):
- return ANDROID_WEBLAYER
- @property
- def default_browser_activity_name(self):
- return 'org.chromium.weblayer.shell.WebLayerShellActivity'
- @property
- def default_finch_seed_path(self):
- return os.path.join(SRC_DIR, 'testing', 'scripts',
- 'variations_smoke_test_data',
- 'variations_seed_stable_weblayer.json')
- @contextlib.contextmanager
- def install_apks(self):
- """Install apks for testing"""
- with super(WebLayerFinchTestCase, self).install_apks(), \
- webview_app.UseWebViewProvider(self._device,
- self.options.webview_provider_apk):
- yield
- def main(args):
- TEST_CASES.update(
- {p.product_name(): p
- for p in [ChromeFinchTestCase, WebViewFinchTestCase,
- WebLayerFinchTestCase]})
- # Unfortunately, there's a circular dependency between the parser made
- # available from `FinchTestCase.add_extra_arguments` and the selection of the
- # correct test case. The workaround is a second parser used in `main` only
- # that shares some arguments with the script adapter parser. The second parser
- # handles --help, so not all arguments are documented. Important arguments
- # added by the script adapter are re-added here for visibility.
- parser = argparse.ArgumentParser()
- FinchTestCase.add_common_arguments(parser)
- parser.add_argument(
- '--isolated-script-test-output', type=str,
- required=False,
- help='path to write test results JSON object to')
- script_common.AddDeviceArguments(parser)
- script_common.AddEnvironmentArguments(parser)
- logging_common.AddLoggingArguments(parser)
- options, _ = parser.parse_known_args(args)
- with get_device(options) as device, \
- TEST_CASES[options.test_case](device) as test_case, \
- test_case.install_apks():
- devil_chromium.Initialize(adb_path=options.adb_path)
- logging_common.InitializeLogging(options)
- # TODO(rmhasan): Best practice in Chromium is to allow users to provide
- # their own adb binary to avoid adb server restarts. We should add a new
- # command line argument to wptrunner so that users can pass the path to
- # their adb binary.
- platform_tools_path = os.path.dirname(devil_env.config.FetchPath('adb'))
- os.environ['PATH'] = os.pathsep.join([platform_tools_path] +
- os.environ['PATH'].split(os.pathsep))
- test_results_dict = OrderedDict({'version': 3, 'interrupted': False,
- 'num_failures_by_type': {}, 'tests': {}})
- if test_case.product_name() == 'webview':
- ret = test_case.run_tests('without_finch_seed', test_results_dict)
- test_case.install_seed()
- ret |= test_case.run_tests('with_finch_seed', test_results_dict)
- # WebView needs several restarts to fetch and load a new finch seed
- # TODO(b/187185389): Figure out why the first restart is needed
- ret |= test_case.run_tests('extra_restart', test_results_dict,
- test_case.finch_seed_download_args())
- # Restart webview+shell to fetch new seed to variations_seed_new
- ret |= test_case.run_tests('fetch_new_seed_restart', test_results_dict,
- test_case.finch_seed_download_args())
- # Restart webview+shell to copy from
- # variations_seed_new to variations_seed
- ret |= test_case.run_tests('load_new_seed_restart', test_results_dict,
- test_case.finch_seed_download_args())
- else:
- test_case.install_seed()
- ret = test_case.run_tests('with_finch_seed', test_results_dict)
- # Clears out the finch seed. Need to run finch_seed tests first.
- # See crbug/1305430
- device.ClearApplicationState(test_case.browser_package_name)
- ret |= test_case.run_tests('without_finch_seed', test_results_dict)
- test_results_dict['seconds_since_epoch'] = int(time.time())
- test_results_dict['path_delimiter'] = '/'
- with open(test_case.options.isolated_script_test_output, 'w') as json_out:
- json_out.write(json.dumps(test_results_dict, indent=4))
- if not test_case.new_seed_downloaded():
- raise Exception('A new seed was not downloaded')
- # Return zero exit code if tests pass
- return ret
- def main_compile_targets(args):
- json.dump([], args.output)
- if __name__ == '__main__':
- if 'compile_targets' in sys.argv:
- funcs = {
- 'run': None,
- 'compile_targets': main_compile_targets,
- }
- sys.exit(common.run_script(sys.argv[1:], funcs))
- sys.exit(main(sys.argv[1:]))
|