123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422 |
- #!/usr/bin/env vpython3
- #
- # Copyright 2018 The Chromium Authors. All rights reserved.
- # Use of this source code is governed by a BSD-style license that can be
- # found in the LICENSE file.
- """Deploys and runs a test package on a Fuchsia target."""
- import argparse
- import logging
- import os
- import shutil
- import sys
- import tempfile
- import ffx_session
- from common_args import AddCommonArgs, AddTargetSpecificArgs, \
- ConfigureLogging, GetDeploymentTargetForArgs
- from net_test_server import SetupTestServer
- from run_test_package import RunTestPackage, RunTestPackageArgs
- from runner_exceptions import HandleExceptionAndReturnExitCode
- sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__),
- 'test')))
- from compatible_utils import map_filter_file_to_package_file
- DEFAULT_TEST_SERVER_CONCURRENCY = 4
- TEST_DATA_DIR = '/tmp'
- TEST_FILTER_PATH = TEST_DATA_DIR + '/test_filter.txt'
- TEST_LLVM_PROFILE_DIR = 'llvm-profile'
- TEST_PERF_RESULT_FILE = 'test_perf_summary.json'
- TEST_RESULT_FILE = 'test_summary.json'
- TEST_REALM_NAME = 'chromium_tests'
- FILTER_DIR = 'testing/buildbot/filters'
- class TestOutputs(object):
- """An abstract base class for extracting outputs generated by a test."""
- def __init__(self):
- pass
- def __enter__(self):
- return self
- def __exit__(self, exc_type, exc_val, exc_tb):
- return False
- def GetFfxSession(self):
- raise NotImplementedError()
- def GetDevicePath(self, path):
- """Returns an absolute device-local variant of a path."""
- raise NotImplementedError()
- def GetFile(self, glob, destination):
- """Places all files/directories matched by a glob into a destination."""
- raise NotImplementedError()
- def GetCoverageProfiles(self, destination):
- """Places all coverage files from the target into a destination."""
- raise NotImplementedError()
- class TargetTestOutputs(TestOutputs):
- """A TestOutputs implementation for CFv1 tests, where tests emit files into
- /tmp that are retrieved from the device via ssh."""
- def __init__(self, target, package_name, test_realms):
- super(TargetTestOutputs, self).__init__()
- self._target = target
- self._package_name = package_name
- self._test_realms = test_realms
- def GetFfxSession(self):
- return None # ffx is not used to run CFv1 tests.
- def GetDevicePath(self, path):
- return TEST_DATA_DIR + '/' + path
- def GetFile(self, glob, destination):
- """Places all files/directories matched by a glob into a destination."""
- self._target.GetFile(self.GetDevicePath(glob),
- destination,
- for_package=self._package_name,
- for_realms=self._test_realms)
- def GetCoverageProfiles(self, destination):
- # Copy all the files in the profile directory. /* is used instead of
- # recursively copying due to permission issues for the latter.
- self._target.GetFile(self.GetDevicePath(TEST_LLVM_PROFILE_DIR + '/*'),
- destination, None, None)
- class CustomArtifactsTestOutputs(TestOutputs):
- """A TestOutputs implementation for CFv2 tests, where tests emit files into
- /custom_artifacts that are retrieved from the device automatically via ffx."""
- def __init__(self, target):
- super(CustomArtifactsTestOutputs, self).__init__()
- self._target = target
- self._ffx_session_context = ffx_session.FfxSession(target._log_manager)
- self._ffx_session = None
- def __enter__(self):
- self._ffx_session = self._ffx_session_context.__enter__()
- return self
- def __exit__(self, exc_type, exc_val, exc_tb):
- self._ffx_session = None
- self._ffx_session_context.__exit__(exc_type, exc_val, exc_tb)
- return False
- def GetFfxSession(self):
- assert self._ffx_session
- return self._ffx_session
- def GetDevicePath(self, path):
- return '/custom_artifacts/' + path
- def GetOutputDirectory(self):
- return self._ffx_session.get_output_dir()
- def GetFile(self, glob, destination):
- """Places all files/directories matched by a glob into a destination."""
- directory = self._ffx_session.get_custom_artifact_directory()
- if not directory:
- logger.error(
- 'Failed to parse custom artifact directory from test summary output '
- 'files. Not copying %s from the device', glob)
- return
- shutil.copy(os.path.join(directory, glob), destination)
- def GetCoverageProfiles(self, destination):
- # Copy all the files in the profile directory.
- # TODO(https://fxbug.dev/77634): Switch to ffx-based extraction once it is
- # implemented.
- self._target.GetFile(
- '/tmp/test_manager:0/children/debug_data:0/data/' +
- TEST_LLVM_PROFILE_DIR + '/*', destination)
- def MakeTestOutputs(component_version, target, package_name, test_realms):
- if component_version == '2':
- return CustomArtifactsTestOutputs(target)
- return TargetTestOutputs(target, package_name, test_realms)
- def AddTestExecutionArgs(arg_parser):
- test_args = arg_parser.add_argument_group('testing',
- 'Test execution arguments')
- test_args.add_argument('--gtest_filter',
- help='GTest filter to use in place of any default.')
- test_args.add_argument(
- '--gtest_repeat',
- help='GTest repeat value to use. This also disables the '
- 'test launcher timeout.')
- test_args.add_argument(
- '--test-launcher-retry-limit',
- help='Number of times that test suite will retry failing '
- 'tests. This is multiplicative with --gtest_repeat.')
- test_args.add_argument('--test-launcher-print-test-stdio',
- choices=['auto', 'always', 'never'],
- help='Controls when full test output is printed.'
- 'auto means to print it when the test failed.')
- test_args.add_argument('--test-launcher-shard-index',
- type=int,
- default=os.environ.get('GTEST_SHARD_INDEX'),
- help='Index of this instance amongst swarming shards.')
- test_args.add_argument('--test-launcher-total-shards',
- type=int,
- default=os.environ.get('GTEST_TOTAL_SHARDS'),
- help='Total number of swarming shards of this suite.')
- test_args.add_argument('--gtest_break_on_failure',
- action='store_true',
- default=False,
- help='Should GTest break on failure; useful with '
- '--gtest_repeat.')
- test_args.add_argument('--single-process-tests',
- action='store_true',
- default=False,
- help='Runs the tests and the launcher in the same '
- 'process. Useful for debugging.')
- test_args.add_argument('--test-launcher-batch-limit',
- type=int,
- help='Sets the limit of test batch to run in a single '
- 'process.')
- # --test-launcher-filter-file is specified relative to --out-dir,
- # so specifying type=os.path.* will break it.
- test_args.add_argument(
- '--test-launcher-filter-file',
- default=None,
- help='Filter file(s) passed to target test process. Use ";" to separate '
- 'multiple filter files ')
- test_args.add_argument('--test-launcher-jobs',
- type=int,
- help='Sets the number of parallel test jobs.')
- test_args.add_argument('--test-launcher-summary-output',
- help='Where the test launcher will output its json.')
- test_args.add_argument('--enable-test-server',
- action='store_true',
- default=False,
- help='Enable Chrome test server spawner.')
- test_args.add_argument(
- '--test-launcher-bot-mode',
- action='store_true',
- default=False,
- help='Informs the TestLauncher to that it should enable '
- 'special allowances for running on a test bot.')
- test_args.add_argument('--isolated-script-test-output',
- help='If present, store test results on this path.')
- test_args.add_argument(
- '--isolated-script-test-perf-output',
- help='If present, store chartjson results on this path.')
- test_args.add_argument('--use-run',
- dest='use_run_test_component',
- default=True,
- action='store_false',
- help='Run the test package using run rather than '
- 'hermetically using run-test-component.')
- test_args.add_argument(
- '--code-coverage',
- default=False,
- action='store_true',
- help='Gather code coverage information and place it in '
- 'the output directory.')
- test_args.add_argument('--code-coverage-dir',
- default=os.getcwd(),
- help='Directory to place code coverage information. '
- 'Only relevant when --code-coverage set to true. '
- 'Defaults to current directory.')
- test_args.add_argument('--gtest_also_run_disabled_tests',
- default=False,
- action='store_true',
- help='Run tests prefixed with DISABLED_')
- test_args.add_argument('child_args',
- nargs='*',
- help='Arguments for the test process.')
- test_args.add_argument('--use-vulkan',
- help='\'native\', \'swiftshader\' or \'none\'.')
- def main():
- parser = argparse.ArgumentParser()
- AddTestExecutionArgs(parser)
- AddCommonArgs(parser)
- AddTargetSpecificArgs(parser)
- args = parser.parse_args()
- # Flag out_dir is required for tests launched with this script.
- if not args.out_dir:
- raise ValueError("out-dir must be specified.")
- if args.component_version == "2":
- args.use_run_test_component = False
- if (args.code_coverage and args.component_version != "2"
- and not args.use_run_test_component):
- if args.enable_test_server:
- # TODO(1254563): Tests that need access to the test server cannot be run
- # as test component under CFv1. Because code coverage requires it, force
- # the test to run as a test component. It is expected that test that tries
- # to use the external test server will fail.
- args.use_run_test_component = True
- else:
- raise ValueError('Collecting code coverage info requires using '
- 'run-test-component.')
- ConfigureLogging(args)
- child_args = []
- if args.test_launcher_shard_index != None:
- child_args.append(
- '--test-launcher-shard-index=%d' % args.test_launcher_shard_index)
- if args.test_launcher_total_shards != None:
- child_args.append(
- '--test-launcher-total-shards=%d' % args.test_launcher_total_shards)
- if args.single_process_tests:
- child_args.append('--single-process-tests')
- if args.test_launcher_bot_mode:
- child_args.append('--test-launcher-bot-mode')
- if args.test_launcher_batch_limit:
- child_args.append('--test-launcher-batch-limit=%d' %
- args.test_launcher_batch_limit)
- # Only set --test-launcher-jobs if the caller specifies it, in general.
- # If the caller enables the test-server then we need to launch the right
- # number of instances to match the maximum number of parallel test jobs, so
- # in that case we set --test-launcher-jobs based on the number of CPU cores
- # specified for the emulator to use.
- test_concurrency = None
- if args.test_launcher_jobs:
- test_concurrency = args.test_launcher_jobs
- elif args.enable_test_server:
- if args.device == 'device':
- test_concurrency = DEFAULT_TEST_SERVER_CONCURRENCY
- else:
- test_concurrency = args.cpu_cores
- if test_concurrency:
- child_args.append('--test-launcher-jobs=%d' % test_concurrency)
- if args.test_launcher_print_test_stdio:
- child_args.append('--test-launcher-print-test-stdio=%s' %
- args.test_launcher_print_test_stdio)
- if args.gtest_filter:
- child_args.append('--gtest_filter=' + args.gtest_filter)
- if args.gtest_repeat:
- child_args.append('--gtest_repeat=' + args.gtest_repeat)
- child_args.append('--test-launcher-timeout=-1')
- if args.test_launcher_retry_limit:
- child_args.append(
- '--test-launcher-retry-limit=' + args.test_launcher_retry_limit)
- if args.gtest_break_on_failure:
- child_args.append('--gtest_break_on_failure')
- if args.gtest_also_run_disabled_tests:
- child_args.append('--gtest_also_run_disabled_tests')
- if args.child_args:
- child_args.extend(args.child_args)
- test_realms = []
- if args.use_run_test_component:
- test_realms = [TEST_REALM_NAME]
- if args.use_vulkan:
- child_args.append('--use-vulkan=' + args.use_vulkan)
- elif args.target_cpu == 'x64':
- # TODO(crbug.com/1261646) Remove once Vulkan is enabled by default.
- child_args.append('--use-vulkan=native')
- else:
- # Use swiftshader on arm64 by default because arm64 bots currently
- # don't support Vulkan emulation.
- child_args.append('--use-vulkan=swiftshader')
- try:
- with GetDeploymentTargetForArgs(args) as target, \
- MakeTestOutputs(args.component_version,
- target,
- args.package_name,
- test_realms) as test_outputs:
- if args.test_launcher_summary_output:
- child_args.append('--test-launcher-summary-output=' +
- test_outputs.GetDevicePath(TEST_RESULT_FILE))
- if args.isolated_script_test_output:
- child_args.append('--isolated-script-test-output=' +
- test_outputs.GetDevicePath(TEST_RESULT_FILE))
- if args.isolated_script_test_perf_output:
- child_args.append('--isolated-script-test-perf-output=' +
- test_outputs.GetDevicePath(TEST_PERF_RESULT_FILE))
- target.Start()
- target.StartSystemLog(args.package)
- if args.test_launcher_filter_file:
- if args.component_version == "2":
- # TODO(crbug.com/1279803): Until one can send file to the device when
- # running a test, filter files must be read from the test package.
- test_launcher_filter_files = map(
- map_filter_file_to_package_file,
- args.test_launcher_filter_file.split(';'))
- child_args.append('--test-launcher-filter-file=' +
- ';'.join(test_launcher_filter_files))
- else:
- test_launcher_filter_files = args.test_launcher_filter_file.split(';')
- with tempfile.NamedTemporaryFile('a+b') as combined_filter_file:
- for filter_file in test_launcher_filter_files:
- with open(filter_file, 'rb') as f:
- combined_filter_file.write(f.read())
- combined_filter_file.seek(0)
- target.PutFile(combined_filter_file.name,
- TEST_FILTER_PATH,
- for_package=args.package_name,
- for_realms=test_realms)
- child_args.append('--test-launcher-filter-file=' + TEST_FILTER_PATH)
- test_server = None
- if args.enable_test_server:
- assert test_concurrency
- (test_server,
- spawner_url_base) = SetupTestServer(target, test_concurrency)
- child_args.append('--remote-test-server-spawner-url-base=' +
- spawner_url_base)
- run_package_args = RunTestPackageArgs.FromCommonArgs(args)
- if args.use_run_test_component:
- run_package_args.test_realm_label = TEST_REALM_NAME
- run_package_args.use_run_test_component = True
- if args.component_version == "2":
- run_package_args.output_directory = test_outputs.GetOutputDirectory()
- returncode = RunTestPackage(target, test_outputs.GetFfxSession(),
- args.package, args.package_name,
- args.component_version, child_args,
- run_package_args)
- if test_server:
- test_server.Stop()
- if args.code_coverage:
- test_outputs.GetCoverageProfiles(args.code_coverage_dir)
- if args.test_launcher_summary_output:
- test_outputs.GetFile(TEST_RESULT_FILE,
- args.test_launcher_summary_output)
- if args.isolated_script_test_output:
- test_outputs.GetFile(TEST_RESULT_FILE, args.isolated_script_test_output)
- if args.isolated_script_test_perf_output:
- test_outputs.GetFile(TEST_PERF_RESULT_FILE,
- args.isolated_script_test_perf_output)
- return returncode
- except:
- return HandleExceptionAndReturnExitCode()
- if __name__ == '__main__':
- sys.exit(main())
|