1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345 |
- #!/usr/bin/env vpython3
- #
- # Copyright 2013 The Chromium Authors. All rights reserved.
- # Use of this source code is governed by a BSD-style license that can be
- # found in the LICENSE file.
- """Runs all types of tests from one unified interface."""
- from __future__ import absolute_import
- import argparse
- import collections
- import contextlib
- import io
- import itertools
- import logging
- import os
- import re
- import shlex
- import shutil
- import signal
- import sys
- import tempfile
- import threading
- import traceback
- import unittest
- # Import _strptime before threaded code. datetime.datetime.strptime is
- # threadsafe except for the initial import of the _strptime module.
- # See http://crbug.com/724524 and https://bugs.python.org/issue7980.
- import _strptime # pylint: disable=unused-import
- # pylint: disable=ungrouped-imports
- from pylib.constants import host_paths
- if host_paths.DEVIL_PATH not in sys.path:
- sys.path.append(host_paths.DEVIL_PATH)
- from devil import base_error
- from devil.utils import reraiser_thread
- from devil.utils import run_tests_helper
- from pylib import constants
- from pylib.base import base_test_result
- from pylib.base import environment_factory
- from pylib.base import output_manager
- from pylib.base import output_manager_factory
- from pylib.base import test_instance_factory
- from pylib.base import test_run_factory
- from pylib.results import json_results
- from pylib.results import report_results
- from pylib.results.presentation import test_results_presentation
- from pylib.utils import local_utils
- from pylib.utils import logdog_helper
- from pylib.utils import logging_utils
- from pylib.utils import test_filter
- from py_utils import contextlib_ext
- from lib.results import result_sink # pylint: disable=import-error
- _DEVIL_STATIC_CONFIG_FILE = os.path.abspath(os.path.join(
- host_paths.DIR_SOURCE_ROOT, 'build', 'android', 'devil_config.json'))
- _RERUN_FAILED_TESTS_FILE = 'rerun_failed_tests.filter'
- def _RealPath(arg):
- if arg.startswith('//'):
- arg = os.path.abspath(os.path.join(host_paths.DIR_SOURCE_ROOT,
- arg[2:].replace('/', os.sep)))
- return os.path.realpath(arg)
- def AddTestLauncherOptions(parser):
- """Adds arguments mirroring //base/test/launcher.
- Args:
- parser: The parser to which arguments should be added.
- Returns:
- The given parser.
- """
- parser.add_argument(
- '--test-launcher-retry-limit',
- '--test_launcher_retry_limit',
- '--num_retries', '--num-retries',
- '--isolated-script-test-launcher-retry-limit',
- dest='num_retries', type=int, default=2,
- help='Number of retries for a test before '
- 'giving up (default: %(default)s).')
- parser.add_argument(
- '--test-launcher-summary-output',
- '--json-results-file',
- dest='json_results_file', type=os.path.realpath,
- help='If set, will dump results in JSON form to the specified file. '
- 'Note that this will also trigger saving per-test logcats to '
- 'logdog.')
- parser.add_argument(
- '--test-launcher-shard-index',
- type=int, default=os.environ.get('GTEST_SHARD_INDEX', 0),
- help='Index of the external shard to run.')
- parser.add_argument(
- '--test-launcher-total-shards',
- type=int, default=os.environ.get('GTEST_TOTAL_SHARDS', 1),
- help='Total number of external shards.')
- test_filter.AddFilterOptions(parser)
- return parser
- def AddCommandLineOptions(parser):
- """Adds arguments to support passing command-line flags to the device."""
- parser.add_argument(
- '--device-flags-file',
- type=os.path.realpath,
- help='The relative filepath to a file containing '
- 'command-line flags to set on the device')
- parser.add_argument(
- '--use-apk-under-test-flags-file',
- action='store_true',
- help='Wether to use the flags file for the apk under test. If set, '
- "the filename will be looked up in the APK's PackageInfo.")
- parser.set_defaults(allow_unknown=True)
- parser.set_defaults(command_line_flags=None)
- def AddTracingOptions(parser):
- # TODO(shenghuazhang): Move this into AddCommonOptions once it's supported
- # for all test types.
- parser.add_argument(
- '--trace-output',
- metavar='FILENAME', type=os.path.realpath,
- help='Path to save test_runner trace json output to.')
- parser.add_argument(
- '--trace-all',
- action='store_true',
- help='Whether to trace all function calls.')
- def AddCommonOptions(parser):
- """Adds all common options to |parser|."""
- default_build_type = os.environ.get('BUILDTYPE', 'Debug')
- debug_or_release_group = parser.add_mutually_exclusive_group()
- debug_or_release_group.add_argument(
- '--debug',
- action='store_const', const='Debug', dest='build_type',
- default=default_build_type,
- help='If set, run test suites under out/Debug. '
- 'Default is env var BUILDTYPE or Debug.')
- debug_or_release_group.add_argument(
- '--release',
- action='store_const', const='Release', dest='build_type',
- help='If set, run test suites under out/Release. '
- 'Default is env var BUILDTYPE or Debug.')
- parser.add_argument(
- '--break-on-failure', '--break_on_failure',
- dest='break_on_failure', action='store_true',
- help='Whether to break on failure.')
- # TODO(jbudorick): Remove this once everything has switched to platform
- # mode.
- parser.add_argument(
- '--enable-platform-mode',
- action='store_true',
- help='Run the test scripts in platform mode, which '
- 'conceptually separates the test runner from the '
- '"device" (local or remote, real or emulated) on '
- 'which the tests are running. [experimental]')
- parser.add_argument(
- '-e', '--environment',
- default='local', choices=constants.VALID_ENVIRONMENTS,
- help='Test environment to run in (default: %(default)s).')
- parser.add_argument(
- '--local-output',
- action='store_true',
- help='Whether to archive test output locally and generate '
- 'a local results detail page.')
- parser.add_argument('--list-tests',
- action='store_true',
- help='List available tests and exit.')
- parser.add_argument('--wrapper-script-args',
- help='A string of args that were passed to the wrapper '
- 'script. This should probably not be edited by a '
- 'user as it is passed by the wrapper itself.')
- class FastLocalDevAction(argparse.Action):
- def __call__(self, parser, namespace, values, option_string=None):
- namespace.enable_concurrent_adb = True
- namespace.enable_device_cache = True
- namespace.extract_test_list_from_filter = True
- namespace.local_output = True
- namespace.num_retries = 0
- namespace.skip_clear_data = True
- parser.add_argument(
- '--fast-local-dev',
- type=bool,
- nargs=0,
- action=FastLocalDevAction,
- help='Alias for: --num-retries=0 --enable-device-cache '
- '--enable-concurrent-adb --skip-clear-data '
- '--extract-test-list-from-filter --local-output')
- # TODO(jbudorick): Remove this once downstream bots have switched to
- # api.test_results.
- parser.add_argument(
- '--flakiness-dashboard-server',
- dest='flakiness_dashboard_server',
- help=argparse.SUPPRESS)
- parser.add_argument(
- '--gs-results-bucket',
- help='Google Storage bucket to upload results to.')
- parser.add_argument(
- '--output-directory',
- dest='output_directory', type=os.path.realpath,
- help='Path to the directory in which build files are'
- ' located (must include build type). This will take'
- ' precedence over --debug and --release')
- parser.add_argument(
- '-v', '--verbose',
- dest='verbose_count', default=0, action='count',
- help='Verbose level (multiple times for more)')
- parser.add_argument(
- '--repeat', '--gtest_repeat', '--gtest-repeat',
- '--isolated-script-test-repeat',
- dest='repeat', type=int, default=0,
- help='Number of times to repeat the specified set of tests.')
- # This is currently only implemented for gtests and instrumentation tests.
- parser.add_argument(
- '--gtest_also_run_disabled_tests', '--gtest-also-run-disabled-tests',
- '--isolated-script-test-also-run-disabled-tests',
- dest='run_disabled', action='store_true',
- help='Also run disabled tests if applicable.')
- # These are currently only implemented for gtests.
- parser.add_argument('--isolated-script-test-output',
- help='If present, store test results on this path.')
- parser.add_argument('--isolated-script-test-perf-output',
- help='If present, store chartjson results on this path.')
- AddTestLauncherOptions(parser)
- def ProcessCommonOptions(args):
- """Processes and handles all common options."""
- run_tests_helper.SetLogLevel(args.verbose_count, add_handler=False)
- if args.verbose_count > 0:
- handler = logging_utils.ColorStreamHandler()
- else:
- handler = logging.StreamHandler(sys.stdout)
- handler.setFormatter(run_tests_helper.CustomFormatter())
- logging.getLogger().addHandler(handler)
- constants.SetBuildType(args.build_type)
- if args.output_directory:
- constants.SetOutputDirectory(args.output_directory)
- def AddDeviceOptions(parser):
- """Adds device options to |parser|."""
- parser = parser.add_argument_group('device arguments')
- parser.add_argument(
- '--adb-path',
- type=os.path.realpath,
- help='Specify the absolute path of the adb binary that '
- 'should be used.')
- parser.add_argument('--denylist-file',
- type=os.path.realpath,
- help='Device denylist file.')
- parser.add_argument(
- '-d', '--device', nargs='+',
- dest='test_devices',
- help='Target device(s) for the test suite to run on.')
- parser.add_argument(
- '--enable-concurrent-adb',
- action='store_true',
- help='Run multiple adb commands at the same time, even '
- 'for the same device.')
- parser.add_argument(
- '--enable-device-cache',
- action='store_true',
- help='Cache device state to disk between runs')
- parser.add_argument(
- '--skip-clear-data',
- action='store_true',
- help='Do not wipe app data between tests. Use this to '
- 'speed up local development and never on bots '
- '(increases flakiness)')
- parser.add_argument(
- '--recover-devices',
- action='store_true',
- help='Attempt to recover devices prior to the final retry. Warning: '
- 'this will cause all devices to reboot.')
- parser.add_argument(
- '--tool',
- dest='tool',
- help='Run the test under a tool '
- '(use --tool help to list them)')
- parser.add_argument(
- '--upload-logcats-file',
- action='store_true',
- dest='upload_logcats_file',
- help='Whether to upload logcat file to logdog.')
- logcat_output_group = parser.add_mutually_exclusive_group()
- logcat_output_group.add_argument(
- '--logcat-output-dir', type=os.path.realpath,
- help='If set, will dump logcats recorded during test run to directory. '
- 'File names will be the device ids with timestamps.')
- logcat_output_group.add_argument(
- '--logcat-output-file', type=os.path.realpath,
- help='If set, will merge logcats recorded during test run and dump them '
- 'to the specified file.')
- def AddEmulatorOptions(parser):
- """Adds emulator-specific options to |parser|."""
- parser = parser.add_argument_group('emulator arguments')
- parser.add_argument(
- '--avd-config',
- type=os.path.realpath,
- help='Path to the avd config textpb. '
- '(See //tools/android/avd/proto/ for message definition'
- ' and existing textpb files.)')
- parser.add_argument(
- '--emulator-count',
- type=int,
- default=1,
- help='Number of emulators to use.')
- parser.add_argument(
- '--emulator-window',
- action='store_true',
- default=False,
- help='Enable graphical window display on the emulator.')
- def AddGTestOptions(parser):
- """Adds gtest options to |parser|."""
- parser = parser.add_argument_group('gtest arguments')
- parser.add_argument(
- '--app-data-file',
- action='append', dest='app_data_files',
- help='A file path relative to the app data directory '
- 'that should be saved to the host.')
- parser.add_argument(
- '--app-data-file-dir',
- help='Host directory to which app data files will be'
- ' saved. Used with --app-data-file.')
- parser.add_argument(
- '--enable-xml-result-parsing',
- action='store_true', help=argparse.SUPPRESS)
- parser.add_argument(
- '--executable-dist-dir',
- type=os.path.realpath,
- help="Path to executable's dist directory for native"
- " (non-apk) tests.")
- parser.add_argument(
- '--extract-test-list-from-filter',
- action='store_true',
- help='When a test filter is specified, and the list of '
- 'tests can be determined from it, skip querying the '
- 'device for the list of all tests. Speeds up local '
- 'development, but is not safe to use on bots ('
- 'http://crbug.com/549214')
- parser.add_argument(
- '--gs-test-artifacts-bucket',
- help=('If present, test artifacts will be uploaded to this Google '
- 'Storage bucket.'))
- parser.add_argument(
- '--render-test-output-dir',
- help='If present, store rendering artifacts in this path.')
- parser.add_argument(
- '--runtime-deps-path',
- dest='runtime_deps_path', type=os.path.realpath,
- help='Runtime data dependency file from GN.')
- parser.add_argument(
- '-t', '--shard-timeout',
- dest='shard_timeout', type=int, default=120,
- help='Timeout to wait for each test (default: %(default)s).')
- parser.add_argument(
- '--store-tombstones',
- dest='store_tombstones', action='store_true',
- help='Add tombstones in results if crash.')
- parser.add_argument(
- '-s', '--suite',
- dest='suite_name', nargs='+', metavar='SUITE_NAME', required=True,
- help='Executable name of the test suite to run.')
- parser.add_argument(
- '--test-apk-incremental-install-json',
- type=os.path.realpath,
- help='Path to install json for the test apk.')
- parser.add_argument('--test-launcher-batch-limit',
- dest='test_launcher_batch_limit',
- type=int,
- help='The max number of tests to run in a shard. '
- 'Ignores non-positive ints and those greater than '
- 'MAX_SHARDS')
- parser.add_argument(
- '-w', '--wait-for-java-debugger', action='store_true',
- help='Wait for java debugger to attach before running any application '
- 'code. Also disables test timeouts and sets retries=0.')
- parser.add_argument(
- '--coverage-dir',
- type=os.path.realpath,
- help='Directory in which to place all generated coverage files.')
- parser.add_argument(
- '--use-existing-test-data',
- action='store_true',
- help='Do not push new files to the device, instead using existing APK '
- 'and test data. Only use when running the same test for multiple '
- 'iterations.')
- def AddInstrumentationTestOptions(parser):
- """Adds Instrumentation test options to |parser|."""
- parser = parser.add_argument_group('instrumentation arguments')
- parser.add_argument('--additional-apex',
- action='append',
- dest='additional_apexs',
- default=[],
- type=_RealPath,
- help='Additional apex that must be installed on '
- 'the device when the tests are run')
- parser.add_argument(
- '--additional-apk',
- action='append', dest='additional_apks', default=[],
- type=_RealPath,
- help='Additional apk that must be installed on '
- 'the device when the tests are run')
- parser.add_argument('--forced-queryable-additional-apk',
- action='append',
- dest='forced_queryable_additional_apks',
- default=[],
- type=_RealPath,
- help='Configures an additional-apk to be forced '
- 'to be queryable by other APKs.')
- parser.add_argument('--instant-additional-apk',
- action='append',
- dest='instant_additional_apks',
- default=[],
- type=_RealPath,
- help='Configures an additional-apk to be an instant APK')
- parser.add_argument(
- '-A', '--annotation',
- dest='annotation_str',
- help='Comma-separated list of annotations. Run only tests with any of '
- 'the given annotations. An annotation can be either a key or a '
- 'key-values pair. A test that has no annotation is considered '
- '"SmallTest".')
- # TODO(jbudorick): Remove support for name-style APK specification once
- # bots are no longer doing it.
- parser.add_argument(
- '--apk-under-test',
- help='Path or name of the apk under test.')
- parser.add_argument(
- '--store-data-in-app-directory',
- action='store_true',
- help='Store test data in the application\'s data directory. By default '
- 'the test data is stored in the external storage folder.')
- parser.add_argument(
- '--module',
- action='append',
- dest='modules',
- help='Specify Android App Bundle modules to install in addition to the '
- 'base module.')
- parser.add_argument(
- '--fake-module',
- action='append',
- dest='fake_modules',
- help='Specify Android App Bundle modules to fake install in addition to '
- 'the real modules.')
- parser.add_argument(
- '--additional-locale',
- action='append',
- dest='additional_locales',
- help='Specify locales in addition to the device locale to install splits '
- 'for when --apk-under-test is an Android App Bundle.')
- parser.add_argument(
- '--coverage-dir',
- type=os.path.realpath,
- help='Directory in which to place all generated '
- 'Jacoco coverage files.')
- parser.add_argument(
- '--disable-dalvik-asserts',
- dest='set_asserts', action='store_false', default=True,
- help='Removes the dalvik.vm.enableassertions property')
- parser.add_argument(
- '--proguard-mapping-path',
- help='.mapping file to use to Deobfuscate java stack traces in test '
- 'output and logcat.')
- parser.add_argument(
- '-E', '--exclude-annotation',
- dest='exclude_annotation_str',
- help='Comma-separated list of annotations. Exclude tests with these '
- 'annotations.')
- parser.add_argument(
- '--enable-breakpad-dump',
- action='store_true',
- help='Stores any breakpad dumps till the end of the test.')
- parser.add_argument(
- '--replace-system-package',
- type=_RealPath,
- default=None,
- help='Use this apk to temporarily replace a system package with the same '
- 'package name.')
- parser.add_argument(
- '--remove-system-package',
- default=[],
- action='append',
- dest='system_packages_to_remove',
- help='Specifies a system package to remove before testing if it exists '
- 'on the system. WARNING: THIS WILL PERMANENTLY REMOVE THE SYSTEM APP. '
- 'Unlike --replace-system-package, the app will not be restored after '
- 'tests are finished.')
- parser.add_argument(
- '--use-voice-interaction-service',
- help='This can be used to update the voice interaction service to be a '
- 'custom one. This is useful for mocking assistants. eg: '
- 'android.assist.service/.MainInteractionService')
- parser.add_argument(
- '--use-webview-provider',
- type=_RealPath, default=None,
- help='Use this apk as the webview provider during test. '
- 'The original provider will be restored if possible, '
- "on Nougat the provider can't be determined and so "
- 'the system will choose the default provider.')
- parser.add_argument(
- '--runtime-deps-path',
- dest='runtime_deps_path', type=os.path.realpath,
- help='Runtime data dependency file from GN.')
- parser.add_argument(
- '--screenshot-directory',
- dest='screenshot_dir', type=os.path.realpath,
- help='Capture screenshots of test failures')
- parser.add_argument(
- '--shared-prefs-file',
- dest='shared_prefs_file', type=_RealPath,
- help='The relative path to a file containing JSON list of shared '
- 'preference files to edit and how to do so. Example list: '
- '[{'
- ' "package": "com.package.example",'
- ' "filename": "ExampleSettings.xml",'
- ' "set": {'
- ' "boolean_key_in_xml": true,'
- ' "string_key_in_xml": "string_value"'
- ' },'
- ' "remove": ['
- ' "key_in_xml_to_remove"'
- ' ]'
- '}]')
- parser.add_argument(
- '--store-tombstones',
- action='store_true', dest='store_tombstones',
- help='Add tombstones in results if crash.')
- parser.add_argument(
- '--strict-mode',
- dest='strict_mode', default='testing',
- help='StrictMode command-line flag set on the device, '
- 'death/testing to kill the process, off to stop '
- 'checking, flash to flash only. (default: %(default)s)')
- parser.add_argument(
- '--test-apk',
- required=True,
- help='Path or name of the apk containing the tests.')
- parser.add_argument(
- '--test-apk-as-instant',
- action='store_true',
- help='Install the test apk as an instant app. '
- 'Instant apps run in a more restrictive execution environment.')
- parser.add_argument(
- '--test-launcher-batch-limit',
- dest='test_launcher_batch_limit',
- type=int,
- help=('Not actually used for instrumentation tests, but can be used as '
- 'a proxy for determining if the current run is a retry without '
- 'patch.'))
- parser.add_argument(
- '--timeout-scale',
- type=float,
- help='Factor by which timeouts should be scaled.')
- parser.add_argument(
- '--is-unit-test',
- action='store_true',
- help=('Specify the test suite as composed of unit tests, blocking '
- 'certain operations.'))
- parser.add_argument(
- '-w', '--wait-for-java-debugger', action='store_true',
- help='Wait for java debugger to attach before running any application '
- 'code. Also disables test timeouts and sets retries=0.')
- # WPR record mode.
- parser.add_argument('--wpr-enable-record',
- action='store_true',
- default=False,
- help='If true, WPR server runs in record mode.'
- 'otherwise, runs in replay mode.')
- parser.add_argument(
- '--approve-app-links',
- help='Force enables Digital Asset Link verification for the provided '
- 'package and domain, example usage: --approve-app-links '
- 'com.android.package:www.example.com')
- # These arguments are suppressed from the help text because they should
- # only ever be specified by an intermediate script.
- parser.add_argument(
- '--apk-under-test-incremental-install-json',
- help=argparse.SUPPRESS)
- parser.add_argument(
- '--test-apk-incremental-install-json',
- type=os.path.realpath,
- help=argparse.SUPPRESS)
- def AddSkiaGoldTestOptions(parser):
- """Adds Skia Gold test options to |parser|."""
- parser = parser.add_argument_group("Skia Gold arguments")
- parser.add_argument(
- '--code-review-system',
- help='A non-default code review system to pass to pass to Gold, if '
- 'applicable')
- parser.add_argument(
- '--continuous-integration-system',
- help='A non-default continuous integration system to pass to Gold, if '
- 'applicable')
- parser.add_argument(
- '--git-revision', help='The git commit currently being tested.')
- parser.add_argument(
- '--gerrit-issue',
- help='The Gerrit issue this test is being run on, if applicable.')
- parser.add_argument(
- '--gerrit-patchset',
- help='The Gerrit patchset this test is being run on, if applicable.')
- parser.add_argument(
- '--buildbucket-id',
- help='The Buildbucket build ID that this test was triggered from, if '
- 'applicable.')
- local_group = parser.add_mutually_exclusive_group()
- local_group.add_argument(
- '--local-pixel-tests',
- action='store_true',
- default=None,
- help='Specifies to run the Skia Gold pixel tests in local mode. When run '
- 'in local mode, uploading to Gold is disabled and traditional '
- 'generated/golden/diff images are output instead of triage links. '
- 'Running in local mode also implies --no-luci-auth. If both this '
- 'and --no-local-pixel-tests are left unset, the test harness will '
- 'attempt to detect whether it is running on a workstation or not '
- 'and set the options accordingly.')
- local_group.add_argument(
- '--no-local-pixel-tests',
- action='store_false',
- dest='local_pixel_tests',
- help='Specifies to run the Skia Gold pixel tests in non-local (bot) '
- 'mode. When run in this mode, data is actually uploaded to Gold and '
- 'triage links are generated. If both this and --local-pixel-tests '
- 'are left unset, the test harness will attempt to detect whether '
- 'it is running on a workstation or not and set the options '
- 'accordingly.')
- parser.add_argument(
- '--no-luci-auth',
- action='store_true',
- default=False,
- help="Don't use the serve account provided by LUCI for authentication "
- 'with Skia Gold, instead relying on gsutil to be pre-authenticated. '
- 'Meant for testing locally instead of on the bots.')
- parser.add_argument(
- '--bypass-skia-gold-functionality',
- action='store_true',
- default=False,
- help='Bypass all interaction with Skia Gold, effectively disabling the '
- 'image comparison portion of any tests that use Gold. Only meant to be '
- 'used in case a Gold outage occurs and cannot be fixed quickly.')
- def AddJUnitTestOptions(parser):
- """Adds junit test options to |parser|."""
- parser = parser.add_argument_group('junit arguments')
- parser.add_argument(
- '--coverage-on-the-fly',
- action='store_true',
- help='Generate coverage data by Jacoco on-the-fly instrumentation.')
- parser.add_argument(
- '--coverage-dir', type=os.path.realpath,
- help='Directory to store coverage info.')
- parser.add_argument(
- '--package-filter',
- help='Filters tests by package.')
- parser.add_argument(
- '--runner-filter',
- help='Filters tests by runner class. Must be fully qualified.')
- parser.add_argument(
- '--shards',
- default=-1,
- type=int,
- help='Number of shards to run junit tests in parallel on. Only 1 shard '
- 'is supported when test-filter is specified. Values less than 1 will '
- 'use auto select.')
- parser.add_argument(
- '-s', '--test-suite', required=True,
- help='JUnit test suite to run.')
- debug_group = parser.add_mutually_exclusive_group()
- debug_group.add_argument(
- '-w', '--wait-for-java-debugger', action='store_const', const='8701',
- dest='debug_socket', help='Alias for --debug-socket=8701')
- debug_group.add_argument(
- '--debug-socket',
- help='Wait for java debugger to attach at specified socket address '
- 'before running any application code. Also disables test timeouts '
- 'and sets retries=0.')
- # These arguments are for Android Robolectric tests.
- parser.add_argument(
- '--robolectric-runtime-deps-dir',
- help='Path to runtime deps for Robolectric.')
- parser.add_argument(
- '--resource-apk',
- required=True,
- help='Path to .ap_ containing binary resources for Robolectric.')
- def AddLinkerTestOptions(parser):
- parser = parser.add_argument_group('linker arguments')
- parser.add_argument(
- '--test-apk',
- type=os.path.realpath,
- help='Path to the linker test APK.')
- def AddMonkeyTestOptions(parser):
- """Adds monkey test options to |parser|."""
- parser = parser.add_argument_group('monkey arguments')
- parser.add_argument('--browser',
- required=True,
- choices=list(constants.PACKAGE_INFO.keys()),
- metavar='BROWSER',
- help='Browser under test.')
- parser.add_argument(
- '--category',
- nargs='*', dest='categories', default=[],
- help='A list of allowed categories. Monkey will only visit activities '
- 'that are listed with one of the specified categories.')
- parser.add_argument(
- '--event-count',
- default=10000, type=int,
- help='Number of events to generate (default: %(default)s).')
- parser.add_argument(
- '--seed',
- type=int,
- help='Seed value for pseudo-random generator. Same seed value generates '
- 'the same sequence of events. Seed is randomized by default.')
- parser.add_argument(
- '--throttle',
- default=100, type=int,
- help='Delay between events (ms) (default: %(default)s). ')
- def AddPythonTestOptions(parser):
- parser = parser.add_argument_group('python arguments')
- parser.add_argument('-s',
- '--suite',
- dest='suite_name',
- metavar='SUITE_NAME',
- choices=list(constants.PYTHON_UNIT_TEST_SUITES.keys()),
- help='Name of the test suite to run.')
- def _CreateClassToFileNameDict(test_apk):
- """Creates a dict mapping classes to file names from size-info apk."""
- constants.CheckOutputDirectory()
- test_apk_size_info = os.path.join(constants.GetOutDirectory(), 'size-info',
- os.path.basename(test_apk) + '.jar.info')
- class_to_file_dict = {}
- # Some tests such as webview_cts_tests use a separately downloaded apk to run
- # tests. This means the apk may not have been built by the system and hence
- # no size info file exists.
- if not os.path.exists(test_apk_size_info):
- logging.debug('Apk size file not found. %s', test_apk_size_info)
- return class_to_file_dict
- with open(test_apk_size_info, 'r') as f:
- for line in f:
- file_class, file_name = line.rstrip().split(',', 1)
- # Only want files that are not prebuilt.
- if file_name.startswith('../../'):
- class_to_file_dict[file_class] = str(
- file_name.replace('../../', '//', 1))
- return class_to_file_dict
- def _RunPythonTests(args):
- """Subcommand of RunTestsCommand which runs python unit tests."""
- suite_vars = constants.PYTHON_UNIT_TEST_SUITES[args.suite_name]
- suite_path = suite_vars['path']
- suite_test_modules = suite_vars['test_modules']
- sys.path = [suite_path] + sys.path
- try:
- suite = unittest.TestSuite()
- suite.addTests(unittest.defaultTestLoader.loadTestsFromName(m)
- for m in suite_test_modules)
- runner = unittest.TextTestRunner(verbosity=1+args.verbose_count)
- return 0 if runner.run(suite).wasSuccessful() else 1
- finally:
- sys.path = sys.path[1:]
- _DEFAULT_PLATFORM_MODE_TESTS = [
- 'gtest', 'instrumentation', 'junit', 'linker', 'monkey'
- ]
- def RunTestsCommand(args, result_sink_client=None):
- """Checks test type and dispatches to the appropriate function.
- Args:
- args: argparse.Namespace object.
- result_sink_client: A ResultSinkClient object.
- Returns:
- Integer indicated exit code.
- Raises:
- Exception: Unknown command name passed in, or an exception from an
- individual test runner.
- """
- command = args.command
- ProcessCommonOptions(args)
- logging.info('command: %s', ' '.join(sys.argv))
- if args.enable_platform_mode or command in _DEFAULT_PLATFORM_MODE_TESTS:
- return RunTestsInPlatformMode(args, result_sink_client)
- if command == 'python':
- return _RunPythonTests(args)
- raise Exception('Unknown test type.')
- def _SinkTestResult(test_result, test_file_name, result_sink_client):
- """Upload test result to result_sink.
- Args:
- test_result: A BaseTestResult object
- test_file_name: A string representing the file location of the test
- result_sink_client: A ResultSinkClient object
- Returns:
- N/A
- """
- # Some tests put in non utf-8 char as part of the test
- # which breaks uploads, so need to decode and re-encode.
- log_decoded = test_result.GetLog()
- if isinstance(log_decoded, bytes):
- log_decoded = log_decoded.decode('utf-8', 'replace')
- html_artifact = ''
- https_artifacts = []
- for link_name, link_url in sorted(test_result.GetLinks().items()):
- if link_url.startswith('https:'):
- https_artifacts.append('<li><a target="_blank" href=%s>%s</a></li>' %
- (link_url, link_name))
- else:
- logging.info('Skipping non-https link %r (%s) for test %s.', link_name,
- link_url, test_result.GetName())
- if https_artifacts:
- html_artifact += '<ul>%s</ul>' % '\n'.join(https_artifacts)
- result_sink_client.Post(test_result.GetName(),
- test_result.GetType(),
- test_result.GetDuration(),
- log_decoded.encode('utf-8'),
- test_file_name,
- failure_reason=test_result.GetFailureReason(),
- html_artifact=html_artifact)
- _SUPPORTED_IN_PLATFORM_MODE = [
- # TODO(jbudorick): Add support for more test types.
- 'gtest',
- 'instrumentation',
- 'junit',
- 'linker',
- 'monkey',
- ]
- def RunTestsInPlatformMode(args, result_sink_client=None):
- def infra_error(message):
- logging.fatal(message)
- sys.exit(constants.INFRA_EXIT_CODE)
- if args.command not in _SUPPORTED_IN_PLATFORM_MODE:
- infra_error('%s is not yet supported in platform mode' % args.command)
- ### Set up sigterm handler.
- contexts_to_notify_on_sigterm = []
- def unexpected_sigterm(_signum, _frame):
- msg = [
- 'Received SIGTERM. Shutting down.',
- ]
- for live_thread in threading.enumerate():
- # pylint: disable=protected-access
- thread_stack = ''.join(traceback.format_stack(
- sys._current_frames()[live_thread.ident]))
- msg.extend([
- 'Thread "%s" (ident: %s) is currently running:' % (
- live_thread.name, live_thread.ident),
- thread_stack])
- for context in contexts_to_notify_on_sigterm:
- context.ReceivedSigterm()
- infra_error('\n'.join(msg))
- signal.signal(signal.SIGTERM, unexpected_sigterm)
- ### Set up results handling.
- # TODO(jbudorick): Rewrite results handling.
- # all_raw_results is a list of lists of
- # base_test_result.TestRunResults objects. Each instance of
- # TestRunResults contains all test results produced by a single try,
- # while each list of TestRunResults contains all tries in a single
- # iteration.
- all_raw_results = []
- # all_iteration_results is a list of base_test_result.TestRunResults
- # objects. Each instance of TestRunResults contains the last test
- # result for each test run in that iteration.
- all_iteration_results = []
- global_results_tags = set()
- json_file = tempfile.NamedTemporaryFile(delete=False)
- json_file.close()
- @contextlib.contextmanager
- def json_finalizer():
- try:
- yield
- finally:
- if args.json_results_file and os.path.exists(json_file.name):
- shutil.move(json_file.name, args.json_results_file)
- elif args.isolated_script_test_output and os.path.exists(json_file.name):
- shutil.move(json_file.name, args.isolated_script_test_output)
- else:
- os.remove(json_file.name)
- @contextlib.contextmanager
- def json_writer():
- try:
- yield
- except Exception:
- global_results_tags.add('UNRELIABLE_RESULTS')
- raise
- finally:
- if args.isolated_script_test_output:
- interrupted = 'UNRELIABLE_RESULTS' in global_results_tags
- json_results.GenerateJsonTestResultFormatFile(all_raw_results,
- interrupted,
- json_file.name,
- indent=2)
- else:
- json_results.GenerateJsonResultsFile(
- all_raw_results,
- json_file.name,
- global_tags=list(global_results_tags),
- indent=2)
- test_class_to_file_name_dict = {}
- # Test Location is only supported for instrumentation tests as it
- # requires the size-info file.
- if test_instance.TestType() == 'instrumentation':
- test_class_to_file_name_dict = _CreateClassToFileNameDict(args.test_apk)
- if result_sink_client:
- for run in all_raw_results:
- for results in run:
- for r in results.GetAll():
- # Matches chrome.page_info.PageInfoViewTest#testChromePage
- match = re.search(r'^(.+\..+)#', r.GetName())
- test_file_name = test_class_to_file_name_dict.get(
- match.group(1)) if match else None
- _SinkTestResult(r, test_file_name, result_sink_client)
- @contextlib.contextmanager
- def upload_logcats_file():
- try:
- yield
- finally:
- if not args.logcat_output_file:
- logging.critical('Cannot upload logcat file: no file specified.')
- elif not os.path.exists(args.logcat_output_file):
- logging.critical("Cannot upload logcat file: file doesn't exist.")
- else:
- with open(args.logcat_output_file) as src:
- dst = logdog_helper.open_text('unified_logcats')
- if dst:
- shutil.copyfileobj(src, dst)
- dst.close()
- logging.critical(
- 'Logcat: %s', logdog_helper.get_viewer_url('unified_logcats'))
- logcats_uploader = contextlib_ext.Optional(
- upload_logcats_file(),
- 'upload_logcats_file' in args and args.upload_logcats_file)
- save_detailed_results = (args.local_output or not local_utils.IsOnSwarming()
- ) and not args.isolated_script_test_output
- ### Set up test objects.
- out_manager = output_manager_factory.CreateOutputManager(args)
- env = environment_factory.CreateEnvironment(
- args, out_manager, infra_error)
- test_instance = test_instance_factory.CreateTestInstance(args, infra_error)
- test_run = test_run_factory.CreateTestRun(env, test_instance, infra_error)
- contexts_to_notify_on_sigterm.append(env)
- contexts_to_notify_on_sigterm.append(test_run)
- if args.list_tests:
- try:
- with out_manager, env, test_instance, test_run:
- test_names = test_run.GetTestsForListing()
- print('There are {} tests:'.format(len(test_names)))
- for n in test_names:
- print(n)
- return 0
- except NotImplementedError:
- sys.stderr.write('Test does not support --list-tests (type={}).\n'.format(
- args.command))
- return 1
- ### Run.
- with out_manager, json_finalizer():
- # |raw_logs_fh| is only used by Robolectric tests.
- raw_logs_fh = io.StringIO() if save_detailed_results else None
- with json_writer(), logcats_uploader, env, test_instance, test_run:
- repetitions = (range(args.repeat +
- 1) if args.repeat >= 0 else itertools.count())
- result_counts = collections.defaultdict(
- lambda: collections.defaultdict(int))
- iteration_count = 0
- for _ in repetitions:
- # raw_results will be populated with base_test_result.TestRunResults by
- # test_run.RunTests(). It is immediately added to all_raw_results so
- # that in the event of an exception, all_raw_results will already have
- # the up-to-date results and those can be written to disk.
- raw_results = []
- all_raw_results.append(raw_results)
- test_run.RunTests(raw_results, raw_logs_fh=raw_logs_fh)
- if not raw_results:
- all_raw_results.pop()
- continue
- iteration_results = base_test_result.TestRunResults()
- for r in reversed(raw_results):
- iteration_results.AddTestRunResults(r)
- all_iteration_results.append(iteration_results)
- iteration_count += 1
- for r in iteration_results.GetAll():
- result_counts[r.GetName()][r.GetType()] += 1
- report_results.LogFull(
- results=iteration_results,
- test_type=test_instance.TestType(),
- test_package=test_run.TestPackage(),
- annotation=getattr(args, 'annotations', None),
- flakiness_server=getattr(args, 'flakiness_dashboard_server',
- None))
- if iteration_results.GetNotPass():
- _LogRerunStatement(iteration_results.GetNotPass(),
- args.wrapper_script_args)
- if args.break_on_failure and not iteration_results.DidRunPass():
- break
- if iteration_count > 1:
- # display summary results
- # only display results for a test if at least one test did not pass
- all_pass = 0
- tot_tests = 0
- for test_name in result_counts:
- tot_tests += 1
- if any(result_counts[test_name][x] for x in (
- base_test_result.ResultType.FAIL,
- base_test_result.ResultType.CRASH,
- base_test_result.ResultType.TIMEOUT,
- base_test_result.ResultType.UNKNOWN)):
- logging.critical(
- '%s: %s',
- test_name,
- ', '.join('%s %s' % (str(result_counts[test_name][i]), i)
- for i in base_test_result.ResultType.GetTypes()))
- else:
- all_pass += 1
- logging.critical('%s of %s tests passed in all %s runs',
- str(all_pass),
- str(tot_tests),
- str(iteration_count))
- if save_detailed_results:
- assert raw_logs_fh
- raw_logs_fh.seek(0)
- raw_logs = raw_logs_fh.read()
- if raw_logs:
- with out_manager.ArchivedTempfile(
- 'raw_logs.txt', 'raw_logs',
- output_manager.Datatype.TEXT) as raw_logs_file:
- raw_logs_file.write(raw_logs)
- logging.critical('RAW LOGS: %s', raw_logs_file.Link())
- with out_manager.ArchivedTempfile(
- 'test_results_presentation.html',
- 'test_results_presentation',
- output_manager.Datatype.HTML) as results_detail_file:
- result_html_string, _, _ = test_results_presentation.result_details(
- json_path=json_file.name,
- test_name=args.command,
- cs_base_url='http://cs.chromium.org',
- local_output=True)
- results_detail_file.write(result_html_string)
- results_detail_file.flush()
- logging.critical('TEST RESULTS: %s', results_detail_file.Link())
- ui_screenshots = test_results_presentation.ui_screenshot_set(
- json_file.name)
- if ui_screenshots:
- with out_manager.ArchivedTempfile(
- 'ui_screenshots.json',
- 'ui_capture',
- output_manager.Datatype.JSON) as ui_screenshot_file:
- ui_screenshot_file.write(ui_screenshots)
- logging.critical('UI Screenshots: %s', ui_screenshot_file.Link())
- return (0 if all(r.DidRunPass() for r in all_iteration_results)
- else constants.ERROR_EXIT_CODE)
- def _LogRerunStatement(failed_tests, wrapper_arg_str):
- """Logs a message that can rerun the failed tests.
- Logs a copy/pasteable message that filters tests so just the failing tests
- are run.
- Args:
- failed_tests: A set of test results that did not pass.
- wrapper_arg_str: A string of args that were passed to the called wrapper
- script.
- """
- rerun_arg_list = []
- try:
- constants.CheckOutputDirectory()
- # constants.CheckOutputDirectory throws bare exceptions.
- except: # pylint: disable=bare-except
- logging.exception('Output directory not found. Unable to generate failing '
- 'test filter file.')
- return
- test_filter_file = os.path.join(os.path.relpath(constants.GetOutDirectory()),
- _RERUN_FAILED_TESTS_FILE)
- arg_list = shlex.split(wrapper_arg_str) if wrapper_arg_str else sys.argv
- index = 0
- while index < len(arg_list):
- arg = arg_list[index]
- # Skip adding the filter=<file> and/or the filter arg as we're replacing
- # it with the new filter arg.
- # This covers --test-filter=, --test-launcher-filter-file=, --gtest-filter=,
- # --test-filter *Foobar.baz, -f *foobar, --package-filter <package>,
- # --runner-filter <runner>.
- if 'filter' in arg or arg == '-f':
- index += 1 if '=' in arg else 2
- continue
- rerun_arg_list.append(arg)
- index += 1
- failed_test_list = [str(t) for t in failed_tests]
- with open(test_filter_file, 'w') as fp:
- for t in failed_test_list:
- # Test result names can have # in them that don't match when applied as
- # a test name filter.
- fp.write('%s\n' % t.replace('#', '.'))
- rerun_arg_list.append('--test-launcher-filter-file=%s' % test_filter_file)
- msg = """
- %d Test(s) failed.
- Rerun failed tests with copy and pastable command:
- %s
- """
- logging.critical(msg, len(failed_tests), shlex.join(rerun_arg_list))
- def DumpThreadStacks(_signal, _frame):
- for thread in threading.enumerate():
- reraiser_thread.LogThreadStack(thread)
- def main():
- signal.signal(signal.SIGUSR1, DumpThreadStacks)
- parser = argparse.ArgumentParser()
- command_parsers = parser.add_subparsers(
- title='test types', dest='command')
- subp = command_parsers.add_parser(
- 'gtest',
- help='googletest-based C++ tests')
- AddCommonOptions(subp)
- AddDeviceOptions(subp)
- AddEmulatorOptions(subp)
- AddGTestOptions(subp)
- AddTracingOptions(subp)
- AddCommandLineOptions(subp)
- subp = command_parsers.add_parser(
- 'instrumentation',
- help='InstrumentationTestCase-based Java tests')
- AddCommonOptions(subp)
- AddDeviceOptions(subp)
- AddEmulatorOptions(subp)
- AddInstrumentationTestOptions(subp)
- AddSkiaGoldTestOptions(subp)
- AddTracingOptions(subp)
- AddCommandLineOptions(subp)
- subp = command_parsers.add_parser(
- 'junit',
- help='JUnit4-based Java tests')
- AddCommonOptions(subp)
- AddJUnitTestOptions(subp)
- subp = command_parsers.add_parser(
- 'linker',
- help='linker tests')
- AddCommonOptions(subp)
- AddDeviceOptions(subp)
- AddEmulatorOptions(subp)
- AddLinkerTestOptions(subp)
- subp = command_parsers.add_parser(
- 'monkey',
- help="tests based on Android's monkey command")
- AddCommonOptions(subp)
- AddDeviceOptions(subp)
- AddEmulatorOptions(subp)
- AddMonkeyTestOptions(subp)
- subp = command_parsers.add_parser(
- 'python',
- help='python tests based on unittest.TestCase')
- AddCommonOptions(subp)
- AddPythonTestOptions(subp)
- args, unknown_args = parser.parse_known_args()
- if unknown_args:
- if hasattr(args, 'allow_unknown') and args.allow_unknown:
- args.command_line_flags = unknown_args
- else:
- parser.error('unrecognized arguments: %s' % ' '.join(unknown_args))
- # --replace-system-package/--remove-system-package has the potential to cause
- # issues if --enable-concurrent-adb is set, so disallow that combination.
- concurrent_adb_enabled = (hasattr(args, 'enable_concurrent_adb')
- and args.enable_concurrent_adb)
- replacing_system_packages = (hasattr(args, 'replace_system_package')
- and args.replace_system_package)
- removing_system_packages = (hasattr(args, 'system_packages_to_remove')
- and args.system_packages_to_remove)
- if (concurrent_adb_enabled
- and (replacing_system_packages or removing_system_packages)):
- parser.error('--enable-concurrent-adb cannot be used with either '
- '--replace-system-package or --remove-system-package')
- # --use-webview-provider has the potential to cause issues if
- # --enable-concurrent-adb is set, so disallow that combination
- if (hasattr(args, 'use_webview_provider') and
- hasattr(args, 'enable_concurrent_adb') and args.use_webview_provider and
- args.enable_concurrent_adb):
- parser.error('--use-webview-provider and --enable-concurrent-adb cannot '
- 'be used together')
- if (getattr(args, 'coverage_on_the_fly', False)
- and not getattr(args, 'coverage_dir', '')):
- parser.error('--coverage-on-the-fly requires --coverage-dir')
- if (hasattr(args, 'debug_socket') or
- (hasattr(args, 'wait_for_java_debugger') and
- args.wait_for_java_debugger)):
- args.num_retries = 0
- # Result-sink may not exist in the environment if rdb stream is not enabled.
- result_sink_client = result_sink.TryInitClient()
- try:
- return RunTestsCommand(args, result_sink_client)
- except base_error.BaseError as e:
- logging.exception('Error occurred.')
- if e.is_infra_error:
- return constants.INFRA_EXIT_CODE
- return constants.ERROR_EXIT_CODE
- except: # pylint: disable=W0702
- logging.exception('Unrecognized error occurred.')
- return constants.ERROR_EXIT_CODE
- if __name__ == '__main__':
- sys.exit(main())
|