123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414 |
- # Copyright 2018 The Chromium Authors. All rights reserved.
- # Use of this source code is governed by a BSD-style license that can be
- # found in the LICENSE file.
- """Test runner for running tests using xcodebuild."""
- import collections
- import logging
- import os
- import subprocess
- import time
- import file_util
- import iossim_util
- import test_apps
- from test_result_util import ResultCollection, TestResult, TestStatus
- import test_runner
- import xcode_log_parser
- LOGGER = logging.getLogger(__name__)
- MAXIMUM_TESTS_PER_SHARD_FOR_RERUN = 20
- XTDEVICE_FOLDER = os.path.expanduser('~/Library/Developer/XCTestDevices')
- def _tests_decided_at_runtime(app_name):
- """Return if tests in app are selected at runtime by app_name.
- This works for suites defined in chromium infra.
- """
- suite_name_fragments = ['ios_chrome_multitasking_eg', '_flaky_eg']
- return any(fragment in app_name for fragment in suite_name_fragments)
- def erase_all_simulators(path=None):
- """Erases all simulator devices.
- Args:
- path: (str) A path with simulators
- """
- command = ['xcrun', 'simctl']
- if path:
- command += ['--set', path]
- LOGGER.info('Erasing all simulators from folder %s.' % path)
- else:
- LOGGER.info('Erasing all simulators.')
- try:
- subprocess.check_call(command + ['erase', 'all'])
- except subprocess.CalledProcessError as e:
- # Logging error instead of throwing so we don't cause failures in case
- # this was indeed failing to clean up.
- message = 'Failed to erase all simulators. Error: %s' % e.output
- LOGGER.error(message)
- def shutdown_all_simulators(path=None):
- """Shutdown all simulator devices.
- Fix for DVTCoreSimulatorAdditionsErrorDomain error.
- Args:
- path: (str) A path with simulators
- """
- command = ['xcrun', 'simctl']
- if path:
- command += ['--set', path]
- LOGGER.info('Shutdown all simulators from folder %s.' % path)
- else:
- LOGGER.info('Shutdown all simulators.')
- try:
- subprocess.check_call(command + ['shutdown', 'all'])
- except subprocess.CalledProcessError as e:
- # Logging error instead of throwing so we don't cause failures in case
- # this was indeed failing to clean up.
- message = 'Failed to shutdown all simulators. Error: %s' % e.output
- LOGGER.error(message)
- def terminate_process(proc):
- """Terminates the process.
- If an error occurs ignore it, just print out a message.
- Args:
- proc: A subprocess.
- """
- try:
- proc.terminate()
- except OSError as ex:
- LOGGER.error('Error while killing a process: %s' % ex)
- class LaunchCommand(object):
- """Stores xcodebuild test launching command."""
- def __init__(self,
- egtests_app,
- udid,
- shards,
- retries,
- readline_timeout,
- out_dir=os.path.basename(os.getcwd()),
- use_clang_coverage=False,
- env=None):
- """Initialize launch command.
- Args:
- egtests_app: (EgtestsApp) An egtests_app to run.
- udid: (str) UDID of a device/simulator.
- shards: (int) A number of shards.
- readline_timeout: (int) Timeout to kill a test process when it doesn't
- have output (in seconds).
- retries: (int) A number of retries.
- out_dir: (str) A folder in which xcodebuild will generate test output.
- By default it is a current directory.
- env: (dict) Environment variables.
- Raises:
- AppNotFoundError: At incorrect egtests_app parameter type.
- """
- if not isinstance(egtests_app, test_apps.EgtestsApp):
- raise test_runner.AppNotFoundError(
- 'Parameter `egtests_app` is not EgtestsApp: %s' % egtests_app)
- self.egtests_app = egtests_app
- self.udid = udid
- self.shards = shards
- self.readline_timeout = readline_timeout
- self.retries = retries
- self.out_dir = out_dir
- self.use_clang_coverage = use_clang_coverage
- self.env = env
- self._log_parser = xcode_log_parser.get_parser()
- def launch_attempt(self, cmd):
- """Launch a process and do logging simultaneously.
- Args:
- cmd: (list[str]) A command to run.
- Returns:
- output - command output as list of strings.
- """
- proc = subprocess.Popen(
- cmd,
- env=self.env,
- stdout=subprocess.PIPE,
- stderr=subprocess.STDOUT,
- )
- return test_runner.print_process_output(proc, timeout=self.readline_timeout)
- def launch(self):
- """Launches tests using xcodebuild."""
- overall_launch_command_result = ResultCollection()
- shards = self.shards
- running_tests = set(self.egtests_app.get_all_tests())
- # total number of attempts is self.retries+1
- for attempt in range(self.retries + 1):
- # Erase all simulators per each attempt
- if iossim_util.is_device_with_udid_simulator(self.udid):
- # kill all running simulators to prevent possible memory leaks
- test_runner.SimulatorTestRunner.kill_simulators()
- shutdown_all_simulators()
- shutdown_all_simulators(XTDEVICE_FOLDER)
- erase_all_simulators()
- erase_all_simulators(XTDEVICE_FOLDER)
- outdir_attempt = os.path.join(self.out_dir, 'attempt_%d' % attempt)
- cmd_list = self.egtests_app.command(outdir_attempt, 'id=%s' % self.udid,
- shards)
- # TODO(crbug.com/914878): add heartbeat logging to xcodebuild_runner.
- LOGGER.info('Start test attempt #%d for command [%s]' % (
- attempt, ' '.join(cmd_list)))
- output = self.launch_attempt(cmd_list)
- if hasattr(self, 'use_clang_coverage') and self.use_clang_coverage:
- # out_dir of LaunchCommand object is the TestRunner out_dir joined with
- # UDID. Use os.path.dirname to retrieve the TestRunner out_dir.
- file_util.move_raw_coverage_data(self.udid,
- os.path.dirname(self.out_dir))
- result = self._log_parser.collect_test_results(outdir_attempt, output)
- tests_selected_at_runtime = _tests_decided_at_runtime(
- self.egtests_app.test_app_path)
- # For most suites, only keep crash status from last attempt since retries
- # will cover any missing tests. For these decided at runtime, retain
- # crashes from all attempts and a dummy "crashed" result will be reported
- # to indicate some tests might never ran.
- # TODO(crbug.com/1235871): Switch back to excluded tests and set
- # |overall_crash| to always True.
- overall_launch_command_result.add_result_collection(
- result, overwrite_crash=not tests_selected_at_runtime)
- result.report_to_result_sink()
- tests_to_include = set()
- # |running_tests| are compiled tests in target intersecting with swarming
- # sharding. For some suites, they are more than what's needed to run.
- if not tests_selected_at_runtime:
- tests_to_include = tests_to_include | (
- running_tests - overall_launch_command_result.expected_tests())
- # Add failed tests from last rounds for runtime decided suites and device
- # suites.
- tests_to_include = (
- tests_to_include
- | overall_launch_command_result.never_expected_tests())
- self.egtests_app.included_tests = list(tests_to_include)
- # Nothing to run in retry.
- if not self.egtests_app.included_tests:
- break
- # If tests are not completed(interrupted or did not start) and there are
- # >= 20 remaining tests, run them with the same number of shards.
- # otherwise re-run with shards=1.
- if (not result.crashed
- # If need to re-run less than 20 tests, 1 shard should be enough.
- or (len(running_tests) -
- len(overall_launch_command_result.expected_tests()) <=
- MAXIMUM_TESTS_PER_SHARD_FOR_RERUN)):
- shards = 1
- return overall_launch_command_result
- class SimulatorParallelTestRunner(test_runner.SimulatorTestRunner):
- """Class for running simulator tests using xCode."""
- def __init__(self, app_path, host_app_path, iossim_path, version, platform,
- out_dir, **kwargs):
- """Initializes a new instance of SimulatorParallelTestRunner class.
- Args:
- app_path: (str) A path to egtests_app.
- host_app_path: (str) A path to the host app for EG2.
- iossim_path: Path to the compiled iossim binary to use.
- Not used, but is required by the base class.
- version: (str) iOS version to run simulator on.
- platform: (str) Name of device.
- out_dir: (str) A directory to emit test data into.
- (Following are potential args in **kwargs)
- release: (bool) Whether this test runner is running for a release build.
- repeat_count: (int) Number of times to run each test (passed to test app).
- retries: (int) A number to retry test run, will re-run only failed tests.
- shards: (int) A number of shards. Default is 1.
- test_cases: (list) List of tests to be included in the test run.
- None or [] to include all tests.
- test_args: List of strings to pass as arguments to the test when
- launching.
- use_clang_coverage: Whether code coverage is enabled in this run.
- env_vars: List of environment variables to pass to the test itself.
- Raises:
- AppNotFoundError: If the given app does not exist.
- PlugInsNotFoundError: If the PlugIns directory does not exist for XCTests.
- XcodeVersionNotFoundError: If the given Xcode version does not exist.
- XCTestPlugInNotFoundError: If the .xctest PlugIn does not exist.
- """
- kwargs['retries'] = kwargs.get('retries') or 0
- super(SimulatorParallelTestRunner,
- self).__init__(app_path, iossim_path, platform, version, out_dir,
- **kwargs)
- self.set_up()
- self.host_app_path = None
- if host_app_path != 'NO_PATH':
- self.host_app_path = os.path.abspath(host_app_path)
- self.logs = collections.OrderedDict()
- self.release = kwargs.get('release') or False
- self.test_results['path_delimiter'] = '/'
- # Do not enable parallel testing when code coverage is enabled, because raw
- # coverage data won't be produced with parallel testing.
- if hasattr(self, 'use_clang_coverage') and self.use_clang_coverage:
- self.shards = 1
- def get_launch_env(self):
- """Returns a dict of environment variables to use to launch the test app.
- Returns:
- A dict of environment variables.
- """
- env = super(test_runner.SimulatorTestRunner, self).get_launch_env()
- env['NSUnbufferedIO'] = 'YES'
- return env
- def get_launch_test_app(self):
- """Returns the proper test_app for the run.
- Returns:
- An implementation of EgtestsApp for the runner.
- """
- return test_apps.EgtestsApp(
- self.app_path,
- included_tests=self.test_cases,
- env_vars=self.env_vars,
- test_args=self.test_args,
- release=self.release,
- repeat_count=self.repeat_count,
- host_app_path=self.host_app_path)
- def launch(self):
- """Launches tests using xcodebuild."""
- test_app = self.get_launch_test_app()
- launch_command = LaunchCommand(
- test_app,
- udid=self.udid,
- shards=self.shards,
- readline_timeout=self.readline_timeout,
- retries=self.retries,
- out_dir=os.path.join(self.out_dir, self.udid),
- use_clang_coverage=(hasattr(self, 'use_clang_coverage') and
- self.use_clang_coverage),
- env=self.get_launch_env())
- overall_result = launch_command.launch()
- # Deletes simulator used in the tests after tests end.
- if iossim_util.is_device_with_udid_simulator(self.udid):
- iossim_util.delete_simulator_by_udid(self.udid)
- # Adds disabled tests to result.
- overall_result.add_and_report_test_names_status(
- launch_command.egtests_app.disabled_tests,
- TestStatus.SKIP,
- expected_status=TestStatus.SKIP,
- test_log='Test disabled.')
- # Adds unexpectedly skipped tests to result if applicable.
- tests_selected_at_runtime = _tests_decided_at_runtime(self.app_path)
- unexpectedly_skipped = []
- # TODO(crbug.com/1048758): For the multitasking or any flaky test suites,
- # |all_tests_to_run| contains more tests than what actually runs.
- if not tests_selected_at_runtime:
- # |all_tests_to_run| takes into consideration that only a subset of tests
- # may have run due to the test sharding logic in run.py.
- all_tests_to_run = set(launch_command.egtests_app.get_all_tests())
- unexpectedly_skipped = list(all_tests_to_run -
- overall_result.all_test_names())
- overall_result.add_and_report_test_names_status(
- unexpectedly_skipped,
- TestStatus.SKIP,
- test_log=('The test is compiled in test target but was unexpectedly '
- 'not run or not finished.'))
- # Add a final crash status to result collection. It will be reported as
- # part of step log in LUCI build.
- if unexpectedly_skipped or overall_result.crashed:
- overall_result.set_crashed_with_prefix(
- crash_message_prefix_line=('Test application crash happened and may '
- 'result in missing tests:'))
- self.test_results = overall_result.standard_json_output(path_delimiter='/')
- self.logs.update(overall_result.test_runner_logs())
- # Return False when:
- # - There are unexpected tests (all results of the tests are unexpected), or
- # - The overall status is crashed and tests are selected at runtime. (i.e.
- # runner is unable to know if all scheduled tests appear in result.)
- return (not overall_result.never_expected_tests() and
- not (tests_selected_at_runtime and overall_result.crashed))
- class DeviceXcodeTestRunner(SimulatorParallelTestRunner,
- test_runner.DeviceTestRunner):
- """Class for running tests on real device using xCode."""
- def __init__(self, app_path, host_app_path, out_dir, **kwargs):
- """Initializes a new instance of DeviceXcodeTestRunner class.
- Args:
- app_path: (str) A path to egtests_app.
- host_app_path: (str) A path to the host app for EG2.
- out_dir: (str) A directory to emit test data into.
- (Following are potential args in **kwargs)
- repeat_count: (int) Number of times to run each test (passed to test app).
- retries: (int) A number to retry test run, will re-run only failed tests.
- test_cases: (list) List of tests to be included in the test run.
- None or [] to include all tests.
- test_args: List of strings to pass as arguments to the test when
- launching.
- env_vars: List of environment variables to pass to the test itself.
- Raises:
- AppNotFoundError: If the given app does not exist.
- DeviceDetectionError: If no device found.
- PlugInsNotFoundError: If the PlugIns directory does not exist for XCTests.
- XcodeVersionNotFoundError: If the given Xcode version does not exist.
- XCTestPlugInNotFoundError: If the .xctest PlugIn does not exist.
- """
- test_runner.DeviceTestRunner.__init__(self, app_path, out_dir, **kwargs)
- self.shards = 1 # For tests on real devices shards=1
- self.version = None
- self.platform = None
- self.host_app_path = None
- if host_app_path != 'NO_PATH':
- self.host_app_path = os.path.abspath(host_app_path)
- self.homedir = ''
- self.release = kwargs.get('release') or False
- self.set_up()
- self.start_time = time.strftime('%Y-%m-%d-%H%M%S', time.localtime())
- self.test_results['path_delimiter'] = '/'
- def set_up(self):
- """Performs setup actions which must occur prior to every test launch."""
- self.uninstall_apps()
- self.wipe_derived_data()
- def tear_down(self):
- """Performs cleanup actions which must occur after every test launch."""
- test_runner.DeviceTestRunner.tear_down(self)
- def launch(self):
- try:
- return super(DeviceXcodeTestRunner, self).launch()
- finally:
- self.tear_down()
|