123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098 |
- # Copyright 2016 The Chromium Authors. All rights reserved.
- # Use of this source code is governed by a BSD-style license that can be
- # found in the LICENSE file.
- """Test runners for iOS."""
- import errno
- import signal
- import sys
- import collections
- import logging
- import os
- import psutil
- import shutil
- import subprocess
- import threading
- import time
- import constants
- import file_util
- import gtest_utils
- import iossim_util
- import test_apps
- from test_result_util import ResultCollection, TestResult, TestStatus
- import test_runner_errors
- import xcode_log_parser
- import xcode_util
- import xctest_utils
- LOGGER = logging.getLogger(__name__)
- DERIVED_DATA = os.path.expanduser('~/Library/Developer/Xcode/DerivedData')
- # TODO(crbug.com/1077277): Move commonly used error classes to
- # test_runner_errors module.
- class TestRunnerError(test_runner_errors.Error):
- """Base class for TestRunner-related errors."""
- pass
- class DeviceError(TestRunnerError):
- """Base class for physical device related errors."""
- pass
- class AppLaunchError(TestRunnerError):
- """The app failed to launch."""
- pass
- class AppNotFoundError(TestRunnerError):
- """The requested app was not found."""
- def __init__(self, app_path):
- super(AppNotFoundError, self).__init__(
- 'App does not exist: %s' % app_path)
- class SystemAlertPresentError(DeviceError):
- """System alert is shown on the device."""
- def __init__(self):
- super(SystemAlertPresentError, self).__init__(
- 'System alert is shown on the device.')
- class DeviceDetectionError(DeviceError):
- """Unexpected number of devices detected."""
- def __init__(self, udids):
- super(DeviceDetectionError, self).__init__(
- 'Expected one device, found %s:\n%s' % (len(udids), '\n'.join(udids)))
- class DeviceRestartError(DeviceError):
- """Error restarting a device."""
- def __init__(self):
- super(DeviceRestartError, self).__init__('Error restarting a device')
- class PlugInsNotFoundError(TestRunnerError):
- """The PlugIns directory was not found."""
- def __init__(self, plugins_dir):
- super(PlugInsNotFoundError, self).__init__(
- 'PlugIns directory does not exist: %s' % plugins_dir)
- class SimulatorNotFoundError(TestRunnerError):
- """The given simulator binary was not found."""
- def __init__(self, iossim_path):
- super(SimulatorNotFoundError, self).__init__(
- 'Simulator does not exist: %s' % iossim_path)
- class TestDataExtractionError(DeviceError):
- """Error extracting test data or crash reports from a device."""
- def __init__(self):
- super(TestDataExtractionError, self).__init__('Failed to extract test data')
- class XcodeVersionNotFoundError(TestRunnerError):
- """The requested version of Xcode was not found."""
- def __init__(self, xcode_version):
- super(XcodeVersionNotFoundError, self).__init__(
- 'Xcode version not found: %s' % xcode_version)
- class XCTestConfigError(TestRunnerError):
- """Error related with XCTest config."""
- def __init__(self, message):
- super(XCTestConfigError,
- self).__init__('Incorrect config related with XCTest: %s' % message)
- class XCTestPlugInNotFoundError(TestRunnerError):
- """The .xctest PlugIn was not found."""
- def __init__(self, xctest_path):
- super(XCTestPlugInNotFoundError, self).__init__(
- 'XCTest not found: %s' % xctest_path)
- class MacToolchainNotFoundError(TestRunnerError):
- """The mac_toolchain is not specified."""
- def __init__(self, mac_toolchain):
- super(MacToolchainNotFoundError, self).__init__(
- 'mac_toolchain is not specified or not found: "%s"' % mac_toolchain)
- class XcodePathNotFoundError(TestRunnerError):
- """The path to Xcode.app is not specified."""
- def __init__(self, xcode_path):
- super(XcodePathNotFoundError, self).__init__(
- 'xcode_path is not specified or does not exist: "%s"' % xcode_path)
- class ShardingDisabledError(TestRunnerError):
- """Temporary error indicating that sharding is not yet implemented."""
- def __init__(self):
- super(ShardingDisabledError, self).__init__(
- 'Sharding has not been implemented!')
- def get_device_ios_version(udid):
- """Gets device iOS version.
- Args:
- udid: (str) iOS device UDID.
- Returns:
- Device UDID.
- """
- return subprocess.check_output(
- ['ideviceinfo', '--udid', udid, '-k',
- 'ProductVersion']).decode('utf-8').strip()
- def defaults_write(d, key, value):
- """Run 'defaults write d key value' command.
- Args:
- d: (str) A dictionary.
- key: (str) A key.
- value: (str) A value.
- """
- LOGGER.info('Run \'defaults write %s %s %s\'' % (d, key, value))
- subprocess.call(['defaults', 'write', d, key, value])
- def defaults_delete(d, key):
- """Run 'defaults delete d key' command.
- Args:
- d: (str) A dictionary.
- key: (str) Key to delete.
- """
- LOGGER.info('Run \'defaults delete %s %s\'' % (d, key))
- subprocess.call(['defaults', 'delete', d, key])
- def terminate_process(proc, proc_name):
- """Terminates the process.
- If an error occurs ignore it, just print out a message.
- Args:
- proc: A subprocess to terminate.
- proc_name: A name of process.
- """
- try:
- LOGGER.info('Killing hung process %s' % proc.pid)
- proc.terminate()
- attempts_to_kill = 3
- ps = psutil.Process(proc.pid)
- for _ in range(attempts_to_kill):
- # Check whether proc.pid process is still alive.
- if ps.is_running():
- LOGGER.info(
- 'Process %s is still alive! %s process might block it.',
- psutil.Process(proc.pid).name(), proc_name)
- running_processes = [
- p for p in psutil.process_iter()
- # Use as_dict() to avoid API changes across versions of psutil.
- if proc_name == p.as_dict(attrs=['name'])['name']]
- if not running_processes:
- LOGGER.debug('There are no running %s processes.', proc_name)
- break
- LOGGER.debug('List of running %s processes: %s'
- % (proc_name, running_processes))
- # Killing running processes with proc_name
- for p in running_processes:
- p.send_signal(signal.SIGKILL)
- psutil.wait_procs(running_processes)
- else:
- LOGGER.info('Process was killed!')
- break
- except OSError as ex:
- LOGGER.info('Error while killing a process: %s' % ex)
- # TODO(crbug.com/1044812): Moved print_process_output to utils class.
- def print_process_output(proc,
- proc_name=None,
- parser=None,
- timeout=constants.READLINE_TIMEOUT):
- """Logs process messages in console and waits until process is done.
- Method waits until no output message and if no message for timeout seconds,
- process will be terminated.
- Args:
- proc: A running process.
- proc_name: (str) A process name that has to be killed
- if no output occurs in specified timeout. Sometimes proc generates
- child process that may block its parent and for such cases
- proc_name refers to the name of child process.
- If proc_name is not specified, process name will be used to kill process.
- parser: A parser.
- timeout: A timeout(in seconds) to subprocess.stdout.readline method.
- """
- out = []
- if not proc_name:
- proc_name = psutil.Process(proc.pid).name()
- while True:
- # subprocess.stdout.readline() might be stuck from time to time
- # and tests fail because of TIMEOUT.
- # Try to fix the issue by adding timer-thread for `timeout` seconds
- # that will kill `frozen` running process if no new line is read
- # and will finish test attempt.
- # If new line appears in timeout, just cancel timer.
- try:
- timer = threading.Timer(timeout, terminate_process, [proc, proc_name])
- timer.start()
- line = proc.stdout.readline()
- finally:
- timer.cancel()
- if not line:
- break
- # |line| will be bytes on python3, and therefore must be decoded prior
- # to rstrip.
- if sys.version_info.major == 3:
- line = line.decode('utf-8')
- line = line.rstrip()
- out.append(line)
- if parser:
- parser.ProcessLine(line)
- LOGGER.info(line)
- sys.stdout.flush()
- if parser:
- parser.Finalize()
- LOGGER.debug('Finished print_process_output.')
- return out
- def get_current_xcode_info():
- """Returns the current Xcode path, version, and build number.
- Returns:
- A dict with 'path', 'version', and 'build' keys.
- 'path': The absolute path to the Xcode installation.
- 'version': The Xcode version.
- 'build': The Xcode build version.
- """
- try:
- out = subprocess.check_output(['xcodebuild',
- '-version']).decode('utf-8').splitlines()
- version, build_version = out[0].split(' ')[-1], out[1].split(' ')[-1]
- path = subprocess.check_output(['xcode-select',
- '--print-path']).decode('utf-8').rstrip()
- except subprocess.CalledProcessError:
- version = build_version = path = None
- return {
- 'path': path,
- 'version': version,
- 'build': build_version,
- }
- class TestRunner(object):
- """Base class containing common functionality."""
- def __init__(self, app_path, out_dir, **kwargs):
- """Initializes a new instance of this class.
- Args:
- app_path: Path to the compiled .app to run.
- out_dir: Directory to emit test data into.
- (Following are potential args in **kwargs)
- env_vars: List of environment variables to pass to the test itself.
- readline_timeout: (int) Timeout to kill a test process when it doesn't
- have output (in seconds).
- repeat_count: Number of times to run each test case (passed to test app).
- retries: Number of times to retry failed test cases in test runner.
- test_args: List of strings to pass as arguments to the test when
- launching.
- test_cases: List of tests to be included in the test run. None or [] to
- include all tests.
- xctest: Whether or not this is an XCTest.
- Raises:
- AppNotFoundError: If the given app does not exist.
- PlugInsNotFoundError: If the PlugIns directory does not exist for XCTests.
- XcodeVersionNotFoundError: If the given Xcode version does not exist.
- XCTestPlugInNotFoundError: If the .xctest PlugIn does not exist.
- """
- app_path = os.path.abspath(app_path)
- if not os.path.exists(app_path):
- raise AppNotFoundError(app_path)
- xcode_info = get_current_xcode_info()
- LOGGER.info('Using Xcode version %s build %s at %s',
- xcode_info['version'],
- xcode_info['build'],
- xcode_info['path'])
- if not os.path.exists(out_dir):
- os.makedirs(out_dir)
- self.app_name = os.path.splitext(os.path.split(app_path)[-1])[0]
- self.app_path = app_path
- self.cfbundleid = test_apps.get_bundle_id(app_path)
- self.env_vars = kwargs.get('env_vars') or []
- self.logs = collections.OrderedDict()
- self.out_dir = out_dir
- self.repeat_count = kwargs.get('repeat_count') or 1
- self.retries = kwargs.get('retries') or 0
- self.shards = kwargs.get('shards') or 1
- self.test_args = kwargs.get('test_args') or []
- self.test_cases = kwargs.get('test_cases') or []
- self.xctest_path = ''
- self.xctest = kwargs.get('xctest') or False
- self.readline_timeout = (
- kwargs.get('readline_timeout') or constants.READLINE_TIMEOUT)
- self.test_results = {}
- self.test_results['version'] = 3
- self.test_results['path_delimiter'] = '.'
- self.test_results['seconds_since_epoch'] = int(time.time())
- # This will be overwritten when the tests complete successfully.
- self.test_results['interrupted'] = True
- if self.xctest:
- plugins_dir = os.path.join(self.app_path, 'PlugIns')
- if not os.path.exists(plugins_dir):
- raise PlugInsNotFoundError(plugins_dir)
- for plugin in os.listdir(plugins_dir):
- if plugin.endswith('.xctest'):
- self.xctest_path = os.path.join(plugins_dir, plugin)
- if not os.path.exists(self.xctest_path):
- raise XCTestPlugInNotFoundError(self.xctest_path)
- # TODO(crbug.com/1185295): Move this method to a utils class.
- @staticmethod
- def remove_proxy_settings():
- """removes any proxy settings which may remain from a previous run."""
- LOGGER.info('Removing any proxy settings.')
- network_services = subprocess.check_output(
- ['networksetup',
- '-listallnetworkservices']).decode('utf-8').strip().split('\n')
- if len(network_services) > 1:
- # We ignore the first line as it is a description of the command's output.
- network_services = network_services[1:]
- for service in network_services:
- # Disabled services have a '*' but calls should not include it
- if service.startswith('*'):
- service = service[1:]
- subprocess.check_call(
- ['networksetup', '-setsocksfirewallproxystate', service, 'off'])
- def get_launch_command(self, test_app, out_dir, destination, shards=1):
- """Returns the command that can be used to launch the test app.
- Args:
- test_app: An app that stores data about test required to run.
- out_dir: (str) A path for results.
- destination: (str) A destination of device/simulator.
- shards: (int) How many shards the tests should be divided into.
- Returns:
- A list of strings forming the command to launch the test.
- """
- raise NotImplementedError
- def get_launch_env(self):
- """Returns a dict of environment variables to use to launch the test app.
- Returns:
- A dict of environment variables.
- """
- return os.environ.copy()
- def get_launch_test_app(self):
- """Returns the proper test_app for the run.
- Returns:
- An implementation of GTestsApp for the current run to execute.
- """
- raise NotImplementedError
- def start_proc(self, cmd):
- """Starts a process with cmd command and os.environ.
- Returns:
- An instance of process.
- """
- return subprocess.Popen(
- cmd,
- env=self.get_launch_env(),
- stdout=subprocess.PIPE,
- stderr=subprocess.STDOUT,
- )
- def shutdown_and_restart(self):
- """Restart a device or relaunch a simulator."""
- pass
- def set_up(self):
- """Performs setup actions which must occur prior to every test launch."""
- raise NotImplementedError
- def tear_down(self):
- """Performs cleanup actions which must occur after every test launch."""
- raise NotImplementedError
- def retrieve_derived_data(self):
- """Retrieves the contents of DerivedData"""
- # DerivedData contains some logs inside workspace-specific directories.
- # Since we don't control the name of the workspace or project, most of
- # the directories are just called "temporary", making it hard to tell
- # which directory we need to retrieve. Instead we just delete the
- # entire contents of this directory before starting and return the
- # entire contents after the test is over.
- if os.path.exists(DERIVED_DATA):
- os.mkdir(os.path.join(self.out_dir, 'DerivedData'))
- derived_data = os.path.join(self.out_dir, 'DerivedData')
- for directory in os.listdir(DERIVED_DATA):
- LOGGER.info('Copying %s directory', directory)
- shutil.move(os.path.join(DERIVED_DATA, directory), derived_data)
- def wipe_derived_data(self):
- """Removes the contents of Xcode's DerivedData directory."""
- if os.path.exists(DERIVED_DATA):
- shutil.rmtree(DERIVED_DATA)
- os.mkdir(DERIVED_DATA)
- def process_xcresult_dir(self):
- """Copies artifacts & diagnostic logs, zips and removes .xcresult dir."""
- # .xcresult dir only exists when using Xcode 11+ and running as XCTest.
- if not xcode_util.using_xcode_11_or_higher() or not self.xctest:
- LOGGER.info('Skip processing xcresult directory.')
- xcresult_paths = []
- # Warning: This piece of code assumes .xcresult folder is directly under
- # self.out_dir. This is true for TestRunner subclasses in this file.
- # xcresult folder path is whatever passed in -resultBundlePath to xcodebuild
- # command appended with '.xcresult' suffix.
- for filename in os.listdir(self.out_dir):
- full_path = os.path.join(self.out_dir, filename)
- if full_path.endswith('.xcresult') and os.path.isdir(full_path):
- xcresult_paths.append(full_path)
- log_parser = xcode_log_parser.get_parser()
- for xcresult in xcresult_paths:
- # This is what was passed in -resultBundlePath to xcodebuild command.
- result_bundle_path = os.path.splitext(xcresult)[0]
- log_parser.copy_artifacts(result_bundle_path)
- log_parser.export_diagnostic_data(result_bundle_path)
- # result_bundle_path is a symlink to xcresult directory.
- if os.path.islink(result_bundle_path):
- os.unlink(result_bundle_path)
- file_util.zip_and_remove_folder(xcresult)
- def run_tests(self, cmd=None):
- """Runs passed-in tests.
- Args:
- cmd: Command to run tests.
- Return:
- out: (list) List of strings of subprocess's output.
- returncode: (int) Return code of subprocess.
- """
- raise NotImplementedError
- def set_sigterm_handler(self, handler):
- """Sets the SIGTERM handler for the test runner.
- This is its own separate function so it can be mocked in tests.
- Args:
- handler: The handler to be called when a SIGTERM is caught
- Returns:
- The previous SIGTERM handler for the test runner.
- """
- LOGGER.debug('Setting sigterm handler.')
- return signal.signal(signal.SIGTERM, handler)
- def handle_sigterm(self, proc):
- """Handles a SIGTERM sent while a test command is executing.
- Will SIGKILL the currently executing test process, then
- attempt to exit gracefully.
- Args:
- proc: The currently executing test process.
- """
- LOGGER.warning('Sigterm caught during test run. Killing test process.')
- proc.kill()
- def _run(self, cmd, shards=1):
- """Runs the specified command, parsing GTest output.
- Args:
- cmd: List of strings forming the command to run.
- Returns:
- TestResult.ResultCollection() object.
- """
- parser = gtest_utils.GTestLogParser()
- # TODO(crbug.com/812705): Implement test sharding for unit tests.
- # TODO(crbug.com/812712): Use thread pool for DeviceTestRunner as well.
- proc = self.start_proc(cmd)
- old_handler = self.set_sigterm_handler(
- lambda _signum, _frame: self.handle_sigterm(proc))
- print_process_output(
- proc, 'xcodebuild', parser, timeout=self.readline_timeout)
- LOGGER.info('Waiting for test process to terminate.')
- proc.wait()
- LOGGER.info('Test process terminated.')
- self.set_sigterm_handler(old_handler)
- sys.stdout.flush()
- LOGGER.debug('Stdout flushed after test process.')
- returncode = proc.returncode
- LOGGER.info('%s returned %s\n', cmd[0], returncode)
- return parser.GetResultCollection()
- def launch(self):
- """Launches the test app."""
- self.set_up()
- # The overall ResultCorrection object holding all runs of all tests in the
- # runner run. It will be updated with each test application launch.
- overall_result = ResultCollection()
- destination = 'id=%s' % self.udid
- test_app = self.get_launch_test_app()
- out_dir = os.path.join(self.out_dir, 'TestResults')
- cmd = self.get_launch_command(test_app, out_dir, destination, self.shards)
- try:
- result = self._run(cmd=cmd, shards=self.shards or 1)
- if result.crashed and not result.crashed_tests():
- # If the app crashed but not during any particular test case, assume
- # it crashed on startup. Try one more time.
- self.shutdown_and_restart()
- LOGGER.warning('Crashed on startup, retrying...\n')
- out_dir = os.path.join(self.out_dir, 'retry_after_crash_on_startup')
- cmd = self.get_launch_command(test_app, out_dir, destination,
- self.shards)
- result = self._run(cmd)
- result.report_to_result_sink()
- if result.crashed and not result.crashed_tests():
- raise AppLaunchError
- overall_result.add_result_collection(result)
- try:
- while result.crashed and result.crashed_tests():
- # If the app crashes during a specific test case, then resume at the
- # next test case. This is achieved by filtering out every test case
- # which has already run.
- LOGGER.warning('Crashed during %s, resuming...\n',
- list(result.crashed_tests()))
- test_app.excluded_tests = list(overall_result.all_test_names())
- # Changing test filter will change selected gtests in this shard.
- # Thus, sharding env vars have to be cleared to ensure needed tests
- # are run. This means there might be duplicate same tests across
- # the shards.
- test_app.remove_gtest_sharding_env_vars()
- retry_out_dir = os.path.join(
- self.out_dir, 'retry_after_crash_%d' % int(time.time()))
- result = self._run(
- self.get_launch_command(test_app, retry_out_dir, destination))
- result.report_to_result_sink()
- # Only keep the last crash status in crash retries in overall crash
- # status.
- overall_result.add_result_collection(result, overwrite_crash=True)
- except OSError as e:
- if e.errno == errno.E2BIG:
- LOGGER.error('Too many test cases to resume.')
- else:
- raise
- # Retry failed test cases.
- test_app.excluded_tests = []
- never_expected_tests = overall_result.never_expected_tests()
- if self.retries and never_expected_tests:
- LOGGER.warning('%s tests failed and will be retried.\n',
- len(never_expected_tests))
- for i in range(self.retries):
- tests_to_retry = list(overall_result.never_expected_tests())
- for test in tests_to_retry:
- LOGGER.info('Retry #%s for %s.\n', i + 1, test)
- test_app.included_tests = [test]
- # Changing test filter will change selected gtests in this shard.
- # Thus, sharding env vars have to be cleared to ensure the test
- # runs when it's the only test in gtest_filter.
- test_app.remove_gtest_sharding_env_vars()
- test_retry_sub_dir = '%s_retry_%d' % (test.replace('/', '_'), i)
- retry_out_dir = os.path.join(self.out_dir, test_retry_sub_dir)
- retry_result = self._run(
- self.get_launch_command(test_app, retry_out_dir, destination))
- if not retry_result.all_test_names():
- retry_result.add_test_result(
- TestResult(
- test,
- TestStatus.SKIP,
- test_log='In single test retry, result of this test '
- 'didn\'t appear in log.'))
- retry_result.report_to_result_sink()
- # No unknown tests might be skipped so do not change
- # |overall_result|'s crash status.
- overall_result.add_result_collection(
- retry_result, ignore_crash=True)
- interrupted = overall_result.crashed
- if interrupted:
- overall_result.set_crashed_with_prefix(
- crash_message_prefix_line='Test application crashed when running '
- 'tests which might have caused some tests never ran or finished.')
- self.test_results = overall_result.standard_json_output()
- self.logs.update(overall_result.test_runner_logs())
- return not overall_result.never_expected_tests() and not interrupted
- finally:
- self.tear_down()
- class SimulatorTestRunner(TestRunner):
- """Class for running tests on iossim."""
- def __init__(self, app_path, iossim_path, platform, version, out_dir,
- **kwargs):
- """Initializes a new instance of this class.
- Args:
- app_path: Path to the compiled .app or .ipa to run.
- iossim_path: Path to the compiled iossim binary to use.
- platform: Name of the platform to simulate. Supported values can be found
- by running "iossim -l". e.g. "iPhone 5s", "iPad Retina".
- version: Version of iOS the platform should be running. Supported values
- can be found by running "iossim -l". e.g. "9.3", "8.2", "7.1".
- out_dir: Directory to emit test data into.
- (Following are potential args in **kwargs)
- env_vars: List of environment variables to pass to the test itself.
- repeat_count: Number of times to run each test case (passed to test app).
- retries: Number of times to retry failed test cases.
- test_args: List of strings to pass as arguments to the test when
- launching.
- test_cases: List of tests to be included in the test run. None or [] to
- include all tests.
- use_clang_coverage: Whether code coverage is enabled in this run.
- wpr_tools_path: Path to pre-installed WPR-related tools
- xctest: Whether or not this is an XCTest.
- Raises:
- AppNotFoundError: If the given app does not exist.
- PlugInsNotFoundError: If the PlugIns directory does not exist for XCTests.
- XcodeVersionNotFoundError: If the given Xcode version does not exist.
- XCTestPlugInNotFoundError: If the .xctest PlugIn does not exist.
- """
- super(SimulatorTestRunner, self).__init__(app_path, out_dir, **kwargs)
- iossim_path = os.path.abspath(iossim_path)
- if not os.path.exists(iossim_path):
- raise SimulatorNotFoundError(iossim_path)
- self.homedir = ''
- self.iossim_path = iossim_path
- self.platform = platform
- self.start_time = None
- self.version = version
- self.shards = kwargs.get('shards') or 1
- self.udid = iossim_util.get_simulator(self.platform, self.version)
- self.use_clang_coverage = kwargs.get('use_clang_coverage') or False
- @staticmethod
- def kill_simulators():
- """Kills all running simulators."""
- try:
- LOGGER.info('Killing simulators.')
- subprocess.check_call([
- 'pkill',
- '-9',
- '-x',
- # The simulator's name varies by Xcode version.
- 'com.apple.CoreSimulator.CoreSimulatorService', # crbug.com/684305
- 'iPhone Simulator', # Xcode 5
- 'iOS Simulator', # Xcode 6
- 'Simulator', # Xcode 7+
- 'simctl', # https://crbug.com/637429
- 'xcodebuild', # https://crbug.com/684305
- ])
- # If a signal was sent, wait for the simulators to actually be killed.
- time.sleep(5)
- except subprocess.CalledProcessError as e:
- if e.returncode != 1:
- # Ignore a 1 exit code (which means there were no simulators to kill).
- raise
- def wipe_simulator(self):
- """Wipes the simulator."""
- iossim_util.wipe_simulator_by_udid(self.udid)
- def get_home_directory(self):
- """Returns the simulator's home directory."""
- return iossim_util.get_home_directory(self.platform, self.version)
- def set_up(self):
- """Performs setup actions which must occur prior to every test launch."""
- self.remove_proxy_settings()
- self.kill_simulators()
- self.wipe_simulator()
- self.wipe_derived_data()
- self.homedir = self.get_home_directory()
- # Crash reports have a timestamp in their file name, formatted as
- # YYYY-MM-DD-HHMMSS. Save the current time in the same format so
- # we can compare and fetch crash reports from this run later on.
- self.start_time = time.strftime('%Y-%m-%d-%H%M%S', time.localtime())
- def extract_test_data(self):
- """Extracts data emitted by the test."""
- if hasattr(self, 'use_clang_coverage') and self.use_clang_coverage:
- file_util.move_raw_coverage_data(self.udid, self.out_dir)
- # Find the Documents directory of the test app. The app directory names
- # don't correspond with any known information, so we have to examine them
- # all until we find one with a matching CFBundleIdentifier.
- apps_dir = os.path.join(
- self.homedir, 'Containers', 'Data', 'Application')
- if os.path.exists(apps_dir):
- for appid_dir in os.listdir(apps_dir):
- docs_dir = os.path.join(apps_dir, appid_dir, 'Documents')
- metadata_plist = os.path.join(
- apps_dir,
- appid_dir,
- '.com.apple.mobile_container_manager.metadata.plist',
- )
- if os.path.exists(docs_dir) and os.path.exists(metadata_plist):
- cfbundleid = subprocess.check_output([
- '/usr/libexec/PlistBuddy',
- '-c',
- 'Print:MCMMetadataIdentifier',
- metadata_plist,
- ]).decode('utf-8').rstrip()
- if cfbundleid == self.cfbundleid:
- shutil.copytree(docs_dir, os.path.join(self.out_dir, 'Documents'))
- return
- def retrieve_crash_reports(self):
- """Retrieves crash reports produced by the test."""
- # A crash report's naming scheme is [app]_[timestamp]_[hostname].crash.
- # e.g. net_unittests_2014-05-13-15-0900_vm1-a1.crash.
- crash_reports_dir = os.path.expanduser(os.path.join(
- '~', 'Library', 'Logs', 'DiagnosticReports'))
- if not os.path.exists(crash_reports_dir):
- return
- for crash_report in os.listdir(crash_reports_dir):
- report_name, ext = os.path.splitext(crash_report)
- if report_name.startswith(self.app_name) and ext == '.crash':
- report_time = report_name[len(self.app_name) + 1:].split('_')[0]
- # The timestamp format in a crash report is big-endian and therefore
- # a straight string comparison works.
- if report_time > self.start_time:
- with open(os.path.join(crash_reports_dir, crash_report)) as f:
- self.logs['crash report (%s)' % report_time] = (
- f.read().splitlines())
- def tear_down(self):
- """Performs cleanup actions which must occur after every test launch."""
- LOGGER.debug('Extracting test data.')
- self.extract_test_data()
- LOGGER.debug('Retrieving crash reports.')
- self.retrieve_crash_reports()
- LOGGER.debug('Retrieving derived data.')
- self.retrieve_derived_data()
- LOGGER.debug('Processing xcresult folder.')
- self.process_xcresult_dir()
- LOGGER.debug('Killing simulators.')
- self.kill_simulators()
- LOGGER.debug('Wiping simulator.')
- self.wipe_simulator()
- LOGGER.debug('Deleting simulator.')
- self.deleteSimulator(self.udid)
- if os.path.exists(self.homedir):
- shutil.rmtree(self.homedir, ignore_errors=True)
- self.homedir = ''
- LOGGER.debug('End of tear_down.')
- def run_tests(self, cmd):
- """Runs passed-in tests. Builds a command and create a simulator to
- run tests.
- Args:
- cmd: A running command.
- Return:
- out: (list) List of strings of subprocess's output.
- returncode: (int) Return code of subprocess.
- """
- proc = self.start_proc(cmd)
- out = print_process_output(
- proc,
- 'xcodebuild',
- xctest_utils.XCTestLogParser(),
- timeout=self.readline_timeout)
- self.deleteSimulator(self.udid)
- return (out, proc.returncode)
- def getSimulator(self):
- """Gets a simulator or creates a new one by device types and runtimes.
- Returns the udid for the created simulator instance.
- Returns:
- An udid of a simulator device.
- """
- return iossim_util.get_simulator(self.platform, self.version)
- def deleteSimulator(self, udid=None):
- """Removes dynamically created simulator devices."""
- if udid:
- iossim_util.delete_simulator_by_udid(udid)
- def get_launch_command(self, test_app, out_dir, destination, shards=1):
- """Returns the command that can be used to launch the test app.
- Args:
- test_app: An app that stores data about test required to run.
- out_dir: (str) A path for results.
- destination: (str) A destination of device/simulator.
- shards: (int) How many shards the tests should be divided into.
- Returns:
- A list of strings forming the command to launch the test.
- """
- return test_app.command(out_dir, destination, shards)
- def get_launch_env(self):
- """Returns a dict of environment variables to use to launch the test app.
- Returns:
- A dict of environment variables.
- """
- env = super(SimulatorTestRunner, self).get_launch_env()
- if self.xctest:
- env['NSUnbufferedIO'] = 'YES'
- return env
- def get_launch_test_app(self):
- """Returns the proper test_app for the run.
- Returns:
- A SimulatorXCTestUnitTestsApp for the current run to execute.
- """
- # Non iOS Chrome users have unit tests not built with XCTest.
- if not self.xctest:
- return test_apps.GTestsApp(
- self.app_path,
- included_tests=self.test_cases,
- env_vars=self.env_vars,
- repeat_count=self.repeat_count,
- test_args=self.test_args)
- return test_apps.SimulatorXCTestUnitTestsApp(
- self.app_path,
- included_tests=self.test_cases,
- env_vars=self.env_vars,
- repeat_count=self.repeat_count,
- test_args=self.test_args)
- class DeviceTestRunner(TestRunner):
- """Class for running tests on devices."""
- def __init__(self, app_path, out_dir, **kwargs):
- """Initializes a new instance of this class.
- Args:
- app_path: Path to the compiled .app to run.
- out_dir: Directory to emit test data into.
- (Following are potential args in **kwargs)
- env_vars: List of environment variables to pass to the test itself.
- repeat_count: Number of times to run each test case (passed to test app).
- restart: Whether or not restart device when test app crashes on startup.
- retries: Number of times to retry failed test cases.
- test_args: List of strings to pass as arguments to the test when
- launching.
- test_cases: List of tests to be included in the test run. None or [] to
- include all tests.
- xctest: Whether or not this is an XCTest.
- Raises:
- AppNotFoundError: If the given app does not exist.
- PlugInsNotFoundError: If the PlugIns directory does not exist for XCTests.
- XcodeVersionNotFoundError: If the given Xcode version does not exist.
- XCTestPlugInNotFoundError: If the .xctest PlugIn does not exist.
- """
- super(DeviceTestRunner, self).__init__(app_path, out_dir, **kwargs)
- self.udid = subprocess.check_output(['idevice_id',
- '--list']).decode('utf-8').rstrip()
- if len(self.udid.splitlines()) != 1:
- raise DeviceDetectionError(self.udid)
- self.restart = kwargs.get('restart') or False
- def uninstall_apps(self):
- """Uninstalls all apps found on the device."""
- for app in self.get_installed_packages():
- cmd = ['ideviceinstaller', '--udid', self.udid, '--uninstall', app]
- print_process_output(self.start_proc(cmd))
- def install_app(self):
- """Installs the app."""
- cmd = ['ideviceinstaller', '--udid', self.udid, '--install', self.app_path]
- print_process_output(self.start_proc(cmd))
- def get_installed_packages(self):
- """Gets a list of installed packages on a device.
- Returns:
- A list of installed packages on a device.
- """
- cmd = ['idevicefs', '--udid', self.udid, 'ls', '@']
- return print_process_output(self.start_proc(cmd))
- def set_up(self):
- """Performs setup actions which must occur prior to every test launch."""
- self.uninstall_apps()
- self.wipe_derived_data()
- self.install_app()
- def extract_test_data(self):
- """Extracts data emitted by the test."""
- cmd = [
- 'idevicefs',
- '--udid', self.udid,
- 'pull',
- '@%s/Documents' % self.cfbundleid,
- os.path.join(self.out_dir, 'Documents'),
- ]
- try:
- print_process_output(self.start_proc(cmd))
- except subprocess.CalledProcessError:
- raise TestDataExtractionError()
- def shutdown_and_restart(self):
- """Restart the device, wait for two minutes."""
- # TODO(crbug.com/760399): swarming bot ios 11 devices turn to be unavailable
- # in a few hours unexpectedly, which is assumed as an ios beta issue. Should
- # remove this method once the bug is fixed.
- if self.restart:
- LOGGER.info('Restarting device, wait for two minutes.')
- try:
- subprocess.check_call(
- ['idevicediagnostics', 'restart', '--udid', self.udid])
- except subprocess.CalledProcessError:
- raise DeviceRestartError()
- time.sleep(120)
- def retrieve_crash_reports(self):
- """Retrieves crash reports produced by the test."""
- logs_dir = os.path.join(self.out_dir, 'Logs')
- os.mkdir(logs_dir)
- cmd = [
- 'idevicecrashreport',
- '--extract',
- '--udid', self.udid,
- logs_dir,
- ]
- try:
- print_process_output(self.start_proc(cmd))
- except subprocess.CalledProcessError:
- # TODO(crbug.com/828951): Raise the exception when the bug is fixed.
- LOGGER.warning('Failed to retrieve crash reports from device.')
- def tear_down(self):
- """Performs cleanup actions which must occur after every test launch."""
- self.retrieve_derived_data()
- self.extract_test_data()
- self.process_xcresult_dir()
- self.retrieve_crash_reports()
- self.uninstall_apps()
- def get_launch_command(self, test_app, out_dir, destination, shards=1):
- """Returns the command that can be used to launch the test app.
- Args:
- test_app: An app that stores data about test required to run.
- out_dir: (str) A path for results.
- destination: (str) A destination of device/simulator.
- shards: (int) How many shards the tests should be divided into.
- Returns:
- A list of strings forming the command to launch the test.
- """
- if self.xctest:
- return test_app.command(out_dir, destination, shards)
- cmd = [
- 'idevice-app-runner',
- '--udid', self.udid,
- '--start', self.cfbundleid,
- ]
- args = []
- if test_app.included_tests or test_app.excluded_tests:
- gtest_filter = test_apps.get_gtest_filter(test_app.included_tests,
- test_app.excluded_tests)
- args.append('--gtest_filter=%s' % gtest_filter)
- for env_var in self.env_vars:
- cmd.extend(['-D', env_var])
- if args or self.test_args:
- cmd.append('--args')
- cmd.extend(self.test_args)
- cmd.extend(args)
- return cmd
- def get_launch_env(self):
- """Returns a dict of environment variables to use to launch the test app.
- Returns:
- A dict of environment variables.
- """
- env = super(DeviceTestRunner, self).get_launch_env()
- if self.xctest:
- env['NSUnbufferedIO'] = 'YES'
- # e.g. ios_web_shell_egtests
- env['APP_TARGET_NAME'] = os.path.splitext(
- os.path.basename(self.app_path))[0]
- # e.g. ios_web_shell_egtests_module
- env['TEST_TARGET_NAME'] = env['APP_TARGET_NAME'] + '_module'
- return env
- def get_launch_test_app(self):
- """Returns the proper test_app for the run.
- Returns:
- A DeviceXCTestUnitTestsApp for the current run to execute.
- """
- # Non iOS Chrome users have unit tests not built with XCTest.
- if not self.xctest:
- return test_apps.GTestsApp(
- self.app_path,
- included_tests=self.test_cases,
- env_vars=self.env_vars,
- repeat_count=self.repeat_count,
- test_args=self.test_args)
- return test_apps.DeviceXCTestUnitTestsApp(
- self.app_path,
- included_tests=self.test_cases,
- env_vars=self.env_vars,
- repeat_count=self.repeat_count,
- test_args=self.test_args)
|