test_runner.py 39 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098
  1. # Copyright 2016 The Chromium Authors. All rights reserved.
  2. # Use of this source code is governed by a BSD-style license that can be
  3. # found in the LICENSE file.
  4. """Test runners for iOS."""
  5. import errno
  6. import signal
  7. import sys
  8. import collections
  9. import logging
  10. import os
  11. import psutil
  12. import shutil
  13. import subprocess
  14. import threading
  15. import time
  16. import constants
  17. import file_util
  18. import gtest_utils
  19. import iossim_util
  20. import test_apps
  21. from test_result_util import ResultCollection, TestResult, TestStatus
  22. import test_runner_errors
  23. import xcode_log_parser
  24. import xcode_util
  25. import xctest_utils
  26. LOGGER = logging.getLogger(__name__)
  27. DERIVED_DATA = os.path.expanduser('~/Library/Developer/Xcode/DerivedData')
  28. # TODO(crbug.com/1077277): Move commonly used error classes to
  29. # test_runner_errors module.
  30. class TestRunnerError(test_runner_errors.Error):
  31. """Base class for TestRunner-related errors."""
  32. pass
  33. class DeviceError(TestRunnerError):
  34. """Base class for physical device related errors."""
  35. pass
  36. class AppLaunchError(TestRunnerError):
  37. """The app failed to launch."""
  38. pass
  39. class AppNotFoundError(TestRunnerError):
  40. """The requested app was not found."""
  41. def __init__(self, app_path):
  42. super(AppNotFoundError, self).__init__(
  43. 'App does not exist: %s' % app_path)
  44. class SystemAlertPresentError(DeviceError):
  45. """System alert is shown on the device."""
  46. def __init__(self):
  47. super(SystemAlertPresentError, self).__init__(
  48. 'System alert is shown on the device.')
  49. class DeviceDetectionError(DeviceError):
  50. """Unexpected number of devices detected."""
  51. def __init__(self, udids):
  52. super(DeviceDetectionError, self).__init__(
  53. 'Expected one device, found %s:\n%s' % (len(udids), '\n'.join(udids)))
  54. class DeviceRestartError(DeviceError):
  55. """Error restarting a device."""
  56. def __init__(self):
  57. super(DeviceRestartError, self).__init__('Error restarting a device')
  58. class PlugInsNotFoundError(TestRunnerError):
  59. """The PlugIns directory was not found."""
  60. def __init__(self, plugins_dir):
  61. super(PlugInsNotFoundError, self).__init__(
  62. 'PlugIns directory does not exist: %s' % plugins_dir)
  63. class SimulatorNotFoundError(TestRunnerError):
  64. """The given simulator binary was not found."""
  65. def __init__(self, iossim_path):
  66. super(SimulatorNotFoundError, self).__init__(
  67. 'Simulator does not exist: %s' % iossim_path)
  68. class TestDataExtractionError(DeviceError):
  69. """Error extracting test data or crash reports from a device."""
  70. def __init__(self):
  71. super(TestDataExtractionError, self).__init__('Failed to extract test data')
  72. class XcodeVersionNotFoundError(TestRunnerError):
  73. """The requested version of Xcode was not found."""
  74. def __init__(self, xcode_version):
  75. super(XcodeVersionNotFoundError, self).__init__(
  76. 'Xcode version not found: %s' % xcode_version)
  77. class XCTestConfigError(TestRunnerError):
  78. """Error related with XCTest config."""
  79. def __init__(self, message):
  80. super(XCTestConfigError,
  81. self).__init__('Incorrect config related with XCTest: %s' % message)
  82. class XCTestPlugInNotFoundError(TestRunnerError):
  83. """The .xctest PlugIn was not found."""
  84. def __init__(self, xctest_path):
  85. super(XCTestPlugInNotFoundError, self).__init__(
  86. 'XCTest not found: %s' % xctest_path)
  87. class MacToolchainNotFoundError(TestRunnerError):
  88. """The mac_toolchain is not specified."""
  89. def __init__(self, mac_toolchain):
  90. super(MacToolchainNotFoundError, self).__init__(
  91. 'mac_toolchain is not specified or not found: "%s"' % mac_toolchain)
  92. class XcodePathNotFoundError(TestRunnerError):
  93. """The path to Xcode.app is not specified."""
  94. def __init__(self, xcode_path):
  95. super(XcodePathNotFoundError, self).__init__(
  96. 'xcode_path is not specified or does not exist: "%s"' % xcode_path)
  97. class ShardingDisabledError(TestRunnerError):
  98. """Temporary error indicating that sharding is not yet implemented."""
  99. def __init__(self):
  100. super(ShardingDisabledError, self).__init__(
  101. 'Sharding has not been implemented!')
  102. def get_device_ios_version(udid):
  103. """Gets device iOS version.
  104. Args:
  105. udid: (str) iOS device UDID.
  106. Returns:
  107. Device UDID.
  108. """
  109. return subprocess.check_output(
  110. ['ideviceinfo', '--udid', udid, '-k',
  111. 'ProductVersion']).decode('utf-8').strip()
  112. def defaults_write(d, key, value):
  113. """Run 'defaults write d key value' command.
  114. Args:
  115. d: (str) A dictionary.
  116. key: (str) A key.
  117. value: (str) A value.
  118. """
  119. LOGGER.info('Run \'defaults write %s %s %s\'' % (d, key, value))
  120. subprocess.call(['defaults', 'write', d, key, value])
  121. def defaults_delete(d, key):
  122. """Run 'defaults delete d key' command.
  123. Args:
  124. d: (str) A dictionary.
  125. key: (str) Key to delete.
  126. """
  127. LOGGER.info('Run \'defaults delete %s %s\'' % (d, key))
  128. subprocess.call(['defaults', 'delete', d, key])
  129. def terminate_process(proc, proc_name):
  130. """Terminates the process.
  131. If an error occurs ignore it, just print out a message.
  132. Args:
  133. proc: A subprocess to terminate.
  134. proc_name: A name of process.
  135. """
  136. try:
  137. LOGGER.info('Killing hung process %s' % proc.pid)
  138. proc.terminate()
  139. attempts_to_kill = 3
  140. ps = psutil.Process(proc.pid)
  141. for _ in range(attempts_to_kill):
  142. # Check whether proc.pid process is still alive.
  143. if ps.is_running():
  144. LOGGER.info(
  145. 'Process %s is still alive! %s process might block it.',
  146. psutil.Process(proc.pid).name(), proc_name)
  147. running_processes = [
  148. p for p in psutil.process_iter()
  149. # Use as_dict() to avoid API changes across versions of psutil.
  150. if proc_name == p.as_dict(attrs=['name'])['name']]
  151. if not running_processes:
  152. LOGGER.debug('There are no running %s processes.', proc_name)
  153. break
  154. LOGGER.debug('List of running %s processes: %s'
  155. % (proc_name, running_processes))
  156. # Killing running processes with proc_name
  157. for p in running_processes:
  158. p.send_signal(signal.SIGKILL)
  159. psutil.wait_procs(running_processes)
  160. else:
  161. LOGGER.info('Process was killed!')
  162. break
  163. except OSError as ex:
  164. LOGGER.info('Error while killing a process: %s' % ex)
  165. # TODO(crbug.com/1044812): Moved print_process_output to utils class.
  166. def print_process_output(proc,
  167. proc_name=None,
  168. parser=None,
  169. timeout=constants.READLINE_TIMEOUT):
  170. """Logs process messages in console and waits until process is done.
  171. Method waits until no output message and if no message for timeout seconds,
  172. process will be terminated.
  173. Args:
  174. proc: A running process.
  175. proc_name: (str) A process name that has to be killed
  176. if no output occurs in specified timeout. Sometimes proc generates
  177. child process that may block its parent and for such cases
  178. proc_name refers to the name of child process.
  179. If proc_name is not specified, process name will be used to kill process.
  180. parser: A parser.
  181. timeout: A timeout(in seconds) to subprocess.stdout.readline method.
  182. """
  183. out = []
  184. if not proc_name:
  185. proc_name = psutil.Process(proc.pid).name()
  186. while True:
  187. # subprocess.stdout.readline() might be stuck from time to time
  188. # and tests fail because of TIMEOUT.
  189. # Try to fix the issue by adding timer-thread for `timeout` seconds
  190. # that will kill `frozen` running process if no new line is read
  191. # and will finish test attempt.
  192. # If new line appears in timeout, just cancel timer.
  193. try:
  194. timer = threading.Timer(timeout, terminate_process, [proc, proc_name])
  195. timer.start()
  196. line = proc.stdout.readline()
  197. finally:
  198. timer.cancel()
  199. if not line:
  200. break
  201. # |line| will be bytes on python3, and therefore must be decoded prior
  202. # to rstrip.
  203. if sys.version_info.major == 3:
  204. line = line.decode('utf-8')
  205. line = line.rstrip()
  206. out.append(line)
  207. if parser:
  208. parser.ProcessLine(line)
  209. LOGGER.info(line)
  210. sys.stdout.flush()
  211. if parser:
  212. parser.Finalize()
  213. LOGGER.debug('Finished print_process_output.')
  214. return out
  215. def get_current_xcode_info():
  216. """Returns the current Xcode path, version, and build number.
  217. Returns:
  218. A dict with 'path', 'version', and 'build' keys.
  219. 'path': The absolute path to the Xcode installation.
  220. 'version': The Xcode version.
  221. 'build': The Xcode build version.
  222. """
  223. try:
  224. out = subprocess.check_output(['xcodebuild',
  225. '-version']).decode('utf-8').splitlines()
  226. version, build_version = out[0].split(' ')[-1], out[1].split(' ')[-1]
  227. path = subprocess.check_output(['xcode-select',
  228. '--print-path']).decode('utf-8').rstrip()
  229. except subprocess.CalledProcessError:
  230. version = build_version = path = None
  231. return {
  232. 'path': path,
  233. 'version': version,
  234. 'build': build_version,
  235. }
  236. class TestRunner(object):
  237. """Base class containing common functionality."""
  238. def __init__(self, app_path, out_dir, **kwargs):
  239. """Initializes a new instance of this class.
  240. Args:
  241. app_path: Path to the compiled .app to run.
  242. out_dir: Directory to emit test data into.
  243. (Following are potential args in **kwargs)
  244. env_vars: List of environment variables to pass to the test itself.
  245. readline_timeout: (int) Timeout to kill a test process when it doesn't
  246. have output (in seconds).
  247. repeat_count: Number of times to run each test case (passed to test app).
  248. retries: Number of times to retry failed test cases in test runner.
  249. test_args: List of strings to pass as arguments to the test when
  250. launching.
  251. test_cases: List of tests to be included in the test run. None or [] to
  252. include all tests.
  253. xctest: Whether or not this is an XCTest.
  254. Raises:
  255. AppNotFoundError: If the given app does not exist.
  256. PlugInsNotFoundError: If the PlugIns directory does not exist for XCTests.
  257. XcodeVersionNotFoundError: If the given Xcode version does not exist.
  258. XCTestPlugInNotFoundError: If the .xctest PlugIn does not exist.
  259. """
  260. app_path = os.path.abspath(app_path)
  261. if not os.path.exists(app_path):
  262. raise AppNotFoundError(app_path)
  263. xcode_info = get_current_xcode_info()
  264. LOGGER.info('Using Xcode version %s build %s at %s',
  265. xcode_info['version'],
  266. xcode_info['build'],
  267. xcode_info['path'])
  268. if not os.path.exists(out_dir):
  269. os.makedirs(out_dir)
  270. self.app_name = os.path.splitext(os.path.split(app_path)[-1])[0]
  271. self.app_path = app_path
  272. self.cfbundleid = test_apps.get_bundle_id(app_path)
  273. self.env_vars = kwargs.get('env_vars') or []
  274. self.logs = collections.OrderedDict()
  275. self.out_dir = out_dir
  276. self.repeat_count = kwargs.get('repeat_count') or 1
  277. self.retries = kwargs.get('retries') or 0
  278. self.shards = kwargs.get('shards') or 1
  279. self.test_args = kwargs.get('test_args') or []
  280. self.test_cases = kwargs.get('test_cases') or []
  281. self.xctest_path = ''
  282. self.xctest = kwargs.get('xctest') or False
  283. self.readline_timeout = (
  284. kwargs.get('readline_timeout') or constants.READLINE_TIMEOUT)
  285. self.test_results = {}
  286. self.test_results['version'] = 3
  287. self.test_results['path_delimiter'] = '.'
  288. self.test_results['seconds_since_epoch'] = int(time.time())
  289. # This will be overwritten when the tests complete successfully.
  290. self.test_results['interrupted'] = True
  291. if self.xctest:
  292. plugins_dir = os.path.join(self.app_path, 'PlugIns')
  293. if not os.path.exists(plugins_dir):
  294. raise PlugInsNotFoundError(plugins_dir)
  295. for plugin in os.listdir(plugins_dir):
  296. if plugin.endswith('.xctest'):
  297. self.xctest_path = os.path.join(plugins_dir, plugin)
  298. if not os.path.exists(self.xctest_path):
  299. raise XCTestPlugInNotFoundError(self.xctest_path)
  300. # TODO(crbug.com/1185295): Move this method to a utils class.
  301. @staticmethod
  302. def remove_proxy_settings():
  303. """removes any proxy settings which may remain from a previous run."""
  304. LOGGER.info('Removing any proxy settings.')
  305. network_services = subprocess.check_output(
  306. ['networksetup',
  307. '-listallnetworkservices']).decode('utf-8').strip().split('\n')
  308. if len(network_services) > 1:
  309. # We ignore the first line as it is a description of the command's output.
  310. network_services = network_services[1:]
  311. for service in network_services:
  312. # Disabled services have a '*' but calls should not include it
  313. if service.startswith('*'):
  314. service = service[1:]
  315. subprocess.check_call(
  316. ['networksetup', '-setsocksfirewallproxystate', service, 'off'])
  317. def get_launch_command(self, test_app, out_dir, destination, shards=1):
  318. """Returns the command that can be used to launch the test app.
  319. Args:
  320. test_app: An app that stores data about test required to run.
  321. out_dir: (str) A path for results.
  322. destination: (str) A destination of device/simulator.
  323. shards: (int) How many shards the tests should be divided into.
  324. Returns:
  325. A list of strings forming the command to launch the test.
  326. """
  327. raise NotImplementedError
  328. def get_launch_env(self):
  329. """Returns a dict of environment variables to use to launch the test app.
  330. Returns:
  331. A dict of environment variables.
  332. """
  333. return os.environ.copy()
  334. def get_launch_test_app(self):
  335. """Returns the proper test_app for the run.
  336. Returns:
  337. An implementation of GTestsApp for the current run to execute.
  338. """
  339. raise NotImplementedError
  340. def start_proc(self, cmd):
  341. """Starts a process with cmd command and os.environ.
  342. Returns:
  343. An instance of process.
  344. """
  345. return subprocess.Popen(
  346. cmd,
  347. env=self.get_launch_env(),
  348. stdout=subprocess.PIPE,
  349. stderr=subprocess.STDOUT,
  350. )
  351. def shutdown_and_restart(self):
  352. """Restart a device or relaunch a simulator."""
  353. pass
  354. def set_up(self):
  355. """Performs setup actions which must occur prior to every test launch."""
  356. raise NotImplementedError
  357. def tear_down(self):
  358. """Performs cleanup actions which must occur after every test launch."""
  359. raise NotImplementedError
  360. def retrieve_derived_data(self):
  361. """Retrieves the contents of DerivedData"""
  362. # DerivedData contains some logs inside workspace-specific directories.
  363. # Since we don't control the name of the workspace or project, most of
  364. # the directories are just called "temporary", making it hard to tell
  365. # which directory we need to retrieve. Instead we just delete the
  366. # entire contents of this directory before starting and return the
  367. # entire contents after the test is over.
  368. if os.path.exists(DERIVED_DATA):
  369. os.mkdir(os.path.join(self.out_dir, 'DerivedData'))
  370. derived_data = os.path.join(self.out_dir, 'DerivedData')
  371. for directory in os.listdir(DERIVED_DATA):
  372. LOGGER.info('Copying %s directory', directory)
  373. shutil.move(os.path.join(DERIVED_DATA, directory), derived_data)
  374. def wipe_derived_data(self):
  375. """Removes the contents of Xcode's DerivedData directory."""
  376. if os.path.exists(DERIVED_DATA):
  377. shutil.rmtree(DERIVED_DATA)
  378. os.mkdir(DERIVED_DATA)
  379. def process_xcresult_dir(self):
  380. """Copies artifacts & diagnostic logs, zips and removes .xcresult dir."""
  381. # .xcresult dir only exists when using Xcode 11+ and running as XCTest.
  382. if not xcode_util.using_xcode_11_or_higher() or not self.xctest:
  383. LOGGER.info('Skip processing xcresult directory.')
  384. xcresult_paths = []
  385. # Warning: This piece of code assumes .xcresult folder is directly under
  386. # self.out_dir. This is true for TestRunner subclasses in this file.
  387. # xcresult folder path is whatever passed in -resultBundlePath to xcodebuild
  388. # command appended with '.xcresult' suffix.
  389. for filename in os.listdir(self.out_dir):
  390. full_path = os.path.join(self.out_dir, filename)
  391. if full_path.endswith('.xcresult') and os.path.isdir(full_path):
  392. xcresult_paths.append(full_path)
  393. log_parser = xcode_log_parser.get_parser()
  394. for xcresult in xcresult_paths:
  395. # This is what was passed in -resultBundlePath to xcodebuild command.
  396. result_bundle_path = os.path.splitext(xcresult)[0]
  397. log_parser.copy_artifacts(result_bundle_path)
  398. log_parser.export_diagnostic_data(result_bundle_path)
  399. # result_bundle_path is a symlink to xcresult directory.
  400. if os.path.islink(result_bundle_path):
  401. os.unlink(result_bundle_path)
  402. file_util.zip_and_remove_folder(xcresult)
  403. def run_tests(self, cmd=None):
  404. """Runs passed-in tests.
  405. Args:
  406. cmd: Command to run tests.
  407. Return:
  408. out: (list) List of strings of subprocess's output.
  409. returncode: (int) Return code of subprocess.
  410. """
  411. raise NotImplementedError
  412. def set_sigterm_handler(self, handler):
  413. """Sets the SIGTERM handler for the test runner.
  414. This is its own separate function so it can be mocked in tests.
  415. Args:
  416. handler: The handler to be called when a SIGTERM is caught
  417. Returns:
  418. The previous SIGTERM handler for the test runner.
  419. """
  420. LOGGER.debug('Setting sigterm handler.')
  421. return signal.signal(signal.SIGTERM, handler)
  422. def handle_sigterm(self, proc):
  423. """Handles a SIGTERM sent while a test command is executing.
  424. Will SIGKILL the currently executing test process, then
  425. attempt to exit gracefully.
  426. Args:
  427. proc: The currently executing test process.
  428. """
  429. LOGGER.warning('Sigterm caught during test run. Killing test process.')
  430. proc.kill()
  431. def _run(self, cmd, shards=1):
  432. """Runs the specified command, parsing GTest output.
  433. Args:
  434. cmd: List of strings forming the command to run.
  435. Returns:
  436. TestResult.ResultCollection() object.
  437. """
  438. parser = gtest_utils.GTestLogParser()
  439. # TODO(crbug.com/812705): Implement test sharding for unit tests.
  440. # TODO(crbug.com/812712): Use thread pool for DeviceTestRunner as well.
  441. proc = self.start_proc(cmd)
  442. old_handler = self.set_sigterm_handler(
  443. lambda _signum, _frame: self.handle_sigterm(proc))
  444. print_process_output(
  445. proc, 'xcodebuild', parser, timeout=self.readline_timeout)
  446. LOGGER.info('Waiting for test process to terminate.')
  447. proc.wait()
  448. LOGGER.info('Test process terminated.')
  449. self.set_sigterm_handler(old_handler)
  450. sys.stdout.flush()
  451. LOGGER.debug('Stdout flushed after test process.')
  452. returncode = proc.returncode
  453. LOGGER.info('%s returned %s\n', cmd[0], returncode)
  454. return parser.GetResultCollection()
  455. def launch(self):
  456. """Launches the test app."""
  457. self.set_up()
  458. # The overall ResultCorrection object holding all runs of all tests in the
  459. # runner run. It will be updated with each test application launch.
  460. overall_result = ResultCollection()
  461. destination = 'id=%s' % self.udid
  462. test_app = self.get_launch_test_app()
  463. out_dir = os.path.join(self.out_dir, 'TestResults')
  464. cmd = self.get_launch_command(test_app, out_dir, destination, self.shards)
  465. try:
  466. result = self._run(cmd=cmd, shards=self.shards or 1)
  467. if result.crashed and not result.crashed_tests():
  468. # If the app crashed but not during any particular test case, assume
  469. # it crashed on startup. Try one more time.
  470. self.shutdown_and_restart()
  471. LOGGER.warning('Crashed on startup, retrying...\n')
  472. out_dir = os.path.join(self.out_dir, 'retry_after_crash_on_startup')
  473. cmd = self.get_launch_command(test_app, out_dir, destination,
  474. self.shards)
  475. result = self._run(cmd)
  476. result.report_to_result_sink()
  477. if result.crashed and not result.crashed_tests():
  478. raise AppLaunchError
  479. overall_result.add_result_collection(result)
  480. try:
  481. while result.crashed and result.crashed_tests():
  482. # If the app crashes during a specific test case, then resume at the
  483. # next test case. This is achieved by filtering out every test case
  484. # which has already run.
  485. LOGGER.warning('Crashed during %s, resuming...\n',
  486. list(result.crashed_tests()))
  487. test_app.excluded_tests = list(overall_result.all_test_names())
  488. # Changing test filter will change selected gtests in this shard.
  489. # Thus, sharding env vars have to be cleared to ensure needed tests
  490. # are run. This means there might be duplicate same tests across
  491. # the shards.
  492. test_app.remove_gtest_sharding_env_vars()
  493. retry_out_dir = os.path.join(
  494. self.out_dir, 'retry_after_crash_%d' % int(time.time()))
  495. result = self._run(
  496. self.get_launch_command(test_app, retry_out_dir, destination))
  497. result.report_to_result_sink()
  498. # Only keep the last crash status in crash retries in overall crash
  499. # status.
  500. overall_result.add_result_collection(result, overwrite_crash=True)
  501. except OSError as e:
  502. if e.errno == errno.E2BIG:
  503. LOGGER.error('Too many test cases to resume.')
  504. else:
  505. raise
  506. # Retry failed test cases.
  507. test_app.excluded_tests = []
  508. never_expected_tests = overall_result.never_expected_tests()
  509. if self.retries and never_expected_tests:
  510. LOGGER.warning('%s tests failed and will be retried.\n',
  511. len(never_expected_tests))
  512. for i in range(self.retries):
  513. tests_to_retry = list(overall_result.never_expected_tests())
  514. for test in tests_to_retry:
  515. LOGGER.info('Retry #%s for %s.\n', i + 1, test)
  516. test_app.included_tests = [test]
  517. # Changing test filter will change selected gtests in this shard.
  518. # Thus, sharding env vars have to be cleared to ensure the test
  519. # runs when it's the only test in gtest_filter.
  520. test_app.remove_gtest_sharding_env_vars()
  521. test_retry_sub_dir = '%s_retry_%d' % (test.replace('/', '_'), i)
  522. retry_out_dir = os.path.join(self.out_dir, test_retry_sub_dir)
  523. retry_result = self._run(
  524. self.get_launch_command(test_app, retry_out_dir, destination))
  525. if not retry_result.all_test_names():
  526. retry_result.add_test_result(
  527. TestResult(
  528. test,
  529. TestStatus.SKIP,
  530. test_log='In single test retry, result of this test '
  531. 'didn\'t appear in log.'))
  532. retry_result.report_to_result_sink()
  533. # No unknown tests might be skipped so do not change
  534. # |overall_result|'s crash status.
  535. overall_result.add_result_collection(
  536. retry_result, ignore_crash=True)
  537. interrupted = overall_result.crashed
  538. if interrupted:
  539. overall_result.set_crashed_with_prefix(
  540. crash_message_prefix_line='Test application crashed when running '
  541. 'tests which might have caused some tests never ran or finished.')
  542. self.test_results = overall_result.standard_json_output()
  543. self.logs.update(overall_result.test_runner_logs())
  544. return not overall_result.never_expected_tests() and not interrupted
  545. finally:
  546. self.tear_down()
  547. class SimulatorTestRunner(TestRunner):
  548. """Class for running tests on iossim."""
  549. def __init__(self, app_path, iossim_path, platform, version, out_dir,
  550. **kwargs):
  551. """Initializes a new instance of this class.
  552. Args:
  553. app_path: Path to the compiled .app or .ipa to run.
  554. iossim_path: Path to the compiled iossim binary to use.
  555. platform: Name of the platform to simulate. Supported values can be found
  556. by running "iossim -l". e.g. "iPhone 5s", "iPad Retina".
  557. version: Version of iOS the platform should be running. Supported values
  558. can be found by running "iossim -l". e.g. "9.3", "8.2", "7.1".
  559. out_dir: Directory to emit test data into.
  560. (Following are potential args in **kwargs)
  561. env_vars: List of environment variables to pass to the test itself.
  562. repeat_count: Number of times to run each test case (passed to test app).
  563. retries: Number of times to retry failed test cases.
  564. test_args: List of strings to pass as arguments to the test when
  565. launching.
  566. test_cases: List of tests to be included in the test run. None or [] to
  567. include all tests.
  568. use_clang_coverage: Whether code coverage is enabled in this run.
  569. wpr_tools_path: Path to pre-installed WPR-related tools
  570. xctest: Whether or not this is an XCTest.
  571. Raises:
  572. AppNotFoundError: If the given app does not exist.
  573. PlugInsNotFoundError: If the PlugIns directory does not exist for XCTests.
  574. XcodeVersionNotFoundError: If the given Xcode version does not exist.
  575. XCTestPlugInNotFoundError: If the .xctest PlugIn does not exist.
  576. """
  577. super(SimulatorTestRunner, self).__init__(app_path, out_dir, **kwargs)
  578. iossim_path = os.path.abspath(iossim_path)
  579. if not os.path.exists(iossim_path):
  580. raise SimulatorNotFoundError(iossim_path)
  581. self.homedir = ''
  582. self.iossim_path = iossim_path
  583. self.platform = platform
  584. self.start_time = None
  585. self.version = version
  586. self.shards = kwargs.get('shards') or 1
  587. self.udid = iossim_util.get_simulator(self.platform, self.version)
  588. self.use_clang_coverage = kwargs.get('use_clang_coverage') or False
  589. @staticmethod
  590. def kill_simulators():
  591. """Kills all running simulators."""
  592. try:
  593. LOGGER.info('Killing simulators.')
  594. subprocess.check_call([
  595. 'pkill',
  596. '-9',
  597. '-x',
  598. # The simulator's name varies by Xcode version.
  599. 'com.apple.CoreSimulator.CoreSimulatorService', # crbug.com/684305
  600. 'iPhone Simulator', # Xcode 5
  601. 'iOS Simulator', # Xcode 6
  602. 'Simulator', # Xcode 7+
  603. 'simctl', # https://crbug.com/637429
  604. 'xcodebuild', # https://crbug.com/684305
  605. ])
  606. # If a signal was sent, wait for the simulators to actually be killed.
  607. time.sleep(5)
  608. except subprocess.CalledProcessError as e:
  609. if e.returncode != 1:
  610. # Ignore a 1 exit code (which means there were no simulators to kill).
  611. raise
  612. def wipe_simulator(self):
  613. """Wipes the simulator."""
  614. iossim_util.wipe_simulator_by_udid(self.udid)
  615. def get_home_directory(self):
  616. """Returns the simulator's home directory."""
  617. return iossim_util.get_home_directory(self.platform, self.version)
  618. def set_up(self):
  619. """Performs setup actions which must occur prior to every test launch."""
  620. self.remove_proxy_settings()
  621. self.kill_simulators()
  622. self.wipe_simulator()
  623. self.wipe_derived_data()
  624. self.homedir = self.get_home_directory()
  625. # Crash reports have a timestamp in their file name, formatted as
  626. # YYYY-MM-DD-HHMMSS. Save the current time in the same format so
  627. # we can compare and fetch crash reports from this run later on.
  628. self.start_time = time.strftime('%Y-%m-%d-%H%M%S', time.localtime())
  629. def extract_test_data(self):
  630. """Extracts data emitted by the test."""
  631. if hasattr(self, 'use_clang_coverage') and self.use_clang_coverage:
  632. file_util.move_raw_coverage_data(self.udid, self.out_dir)
  633. # Find the Documents directory of the test app. The app directory names
  634. # don't correspond with any known information, so we have to examine them
  635. # all until we find one with a matching CFBundleIdentifier.
  636. apps_dir = os.path.join(
  637. self.homedir, 'Containers', 'Data', 'Application')
  638. if os.path.exists(apps_dir):
  639. for appid_dir in os.listdir(apps_dir):
  640. docs_dir = os.path.join(apps_dir, appid_dir, 'Documents')
  641. metadata_plist = os.path.join(
  642. apps_dir,
  643. appid_dir,
  644. '.com.apple.mobile_container_manager.metadata.plist',
  645. )
  646. if os.path.exists(docs_dir) and os.path.exists(metadata_plist):
  647. cfbundleid = subprocess.check_output([
  648. '/usr/libexec/PlistBuddy',
  649. '-c',
  650. 'Print:MCMMetadataIdentifier',
  651. metadata_plist,
  652. ]).decode('utf-8').rstrip()
  653. if cfbundleid == self.cfbundleid:
  654. shutil.copytree(docs_dir, os.path.join(self.out_dir, 'Documents'))
  655. return
  656. def retrieve_crash_reports(self):
  657. """Retrieves crash reports produced by the test."""
  658. # A crash report's naming scheme is [app]_[timestamp]_[hostname].crash.
  659. # e.g. net_unittests_2014-05-13-15-0900_vm1-a1.crash.
  660. crash_reports_dir = os.path.expanduser(os.path.join(
  661. '~', 'Library', 'Logs', 'DiagnosticReports'))
  662. if not os.path.exists(crash_reports_dir):
  663. return
  664. for crash_report in os.listdir(crash_reports_dir):
  665. report_name, ext = os.path.splitext(crash_report)
  666. if report_name.startswith(self.app_name) and ext == '.crash':
  667. report_time = report_name[len(self.app_name) + 1:].split('_')[0]
  668. # The timestamp format in a crash report is big-endian and therefore
  669. # a straight string comparison works.
  670. if report_time > self.start_time:
  671. with open(os.path.join(crash_reports_dir, crash_report)) as f:
  672. self.logs['crash report (%s)' % report_time] = (
  673. f.read().splitlines())
  674. def tear_down(self):
  675. """Performs cleanup actions which must occur after every test launch."""
  676. LOGGER.debug('Extracting test data.')
  677. self.extract_test_data()
  678. LOGGER.debug('Retrieving crash reports.')
  679. self.retrieve_crash_reports()
  680. LOGGER.debug('Retrieving derived data.')
  681. self.retrieve_derived_data()
  682. LOGGER.debug('Processing xcresult folder.')
  683. self.process_xcresult_dir()
  684. LOGGER.debug('Killing simulators.')
  685. self.kill_simulators()
  686. LOGGER.debug('Wiping simulator.')
  687. self.wipe_simulator()
  688. LOGGER.debug('Deleting simulator.')
  689. self.deleteSimulator(self.udid)
  690. if os.path.exists(self.homedir):
  691. shutil.rmtree(self.homedir, ignore_errors=True)
  692. self.homedir = ''
  693. LOGGER.debug('End of tear_down.')
  694. def run_tests(self, cmd):
  695. """Runs passed-in tests. Builds a command and create a simulator to
  696. run tests.
  697. Args:
  698. cmd: A running command.
  699. Return:
  700. out: (list) List of strings of subprocess's output.
  701. returncode: (int) Return code of subprocess.
  702. """
  703. proc = self.start_proc(cmd)
  704. out = print_process_output(
  705. proc,
  706. 'xcodebuild',
  707. xctest_utils.XCTestLogParser(),
  708. timeout=self.readline_timeout)
  709. self.deleteSimulator(self.udid)
  710. return (out, proc.returncode)
  711. def getSimulator(self):
  712. """Gets a simulator or creates a new one by device types and runtimes.
  713. Returns the udid for the created simulator instance.
  714. Returns:
  715. An udid of a simulator device.
  716. """
  717. return iossim_util.get_simulator(self.platform, self.version)
  718. def deleteSimulator(self, udid=None):
  719. """Removes dynamically created simulator devices."""
  720. if udid:
  721. iossim_util.delete_simulator_by_udid(udid)
  722. def get_launch_command(self, test_app, out_dir, destination, shards=1):
  723. """Returns the command that can be used to launch the test app.
  724. Args:
  725. test_app: An app that stores data about test required to run.
  726. out_dir: (str) A path for results.
  727. destination: (str) A destination of device/simulator.
  728. shards: (int) How many shards the tests should be divided into.
  729. Returns:
  730. A list of strings forming the command to launch the test.
  731. """
  732. return test_app.command(out_dir, destination, shards)
  733. def get_launch_env(self):
  734. """Returns a dict of environment variables to use to launch the test app.
  735. Returns:
  736. A dict of environment variables.
  737. """
  738. env = super(SimulatorTestRunner, self).get_launch_env()
  739. if self.xctest:
  740. env['NSUnbufferedIO'] = 'YES'
  741. return env
  742. def get_launch_test_app(self):
  743. """Returns the proper test_app for the run.
  744. Returns:
  745. A SimulatorXCTestUnitTestsApp for the current run to execute.
  746. """
  747. # Non iOS Chrome users have unit tests not built with XCTest.
  748. if not self.xctest:
  749. return test_apps.GTestsApp(
  750. self.app_path,
  751. included_tests=self.test_cases,
  752. env_vars=self.env_vars,
  753. repeat_count=self.repeat_count,
  754. test_args=self.test_args)
  755. return test_apps.SimulatorXCTestUnitTestsApp(
  756. self.app_path,
  757. included_tests=self.test_cases,
  758. env_vars=self.env_vars,
  759. repeat_count=self.repeat_count,
  760. test_args=self.test_args)
  761. class DeviceTestRunner(TestRunner):
  762. """Class for running tests on devices."""
  763. def __init__(self, app_path, out_dir, **kwargs):
  764. """Initializes a new instance of this class.
  765. Args:
  766. app_path: Path to the compiled .app to run.
  767. out_dir: Directory to emit test data into.
  768. (Following are potential args in **kwargs)
  769. env_vars: List of environment variables to pass to the test itself.
  770. repeat_count: Number of times to run each test case (passed to test app).
  771. restart: Whether or not restart device when test app crashes on startup.
  772. retries: Number of times to retry failed test cases.
  773. test_args: List of strings to pass as arguments to the test when
  774. launching.
  775. test_cases: List of tests to be included in the test run. None or [] to
  776. include all tests.
  777. xctest: Whether or not this is an XCTest.
  778. Raises:
  779. AppNotFoundError: If the given app does not exist.
  780. PlugInsNotFoundError: If the PlugIns directory does not exist for XCTests.
  781. XcodeVersionNotFoundError: If the given Xcode version does not exist.
  782. XCTestPlugInNotFoundError: If the .xctest PlugIn does not exist.
  783. """
  784. super(DeviceTestRunner, self).__init__(app_path, out_dir, **kwargs)
  785. self.udid = subprocess.check_output(['idevice_id',
  786. '--list']).decode('utf-8').rstrip()
  787. if len(self.udid.splitlines()) != 1:
  788. raise DeviceDetectionError(self.udid)
  789. self.restart = kwargs.get('restart') or False
  790. def uninstall_apps(self):
  791. """Uninstalls all apps found on the device."""
  792. for app in self.get_installed_packages():
  793. cmd = ['ideviceinstaller', '--udid', self.udid, '--uninstall', app]
  794. print_process_output(self.start_proc(cmd))
  795. def install_app(self):
  796. """Installs the app."""
  797. cmd = ['ideviceinstaller', '--udid', self.udid, '--install', self.app_path]
  798. print_process_output(self.start_proc(cmd))
  799. def get_installed_packages(self):
  800. """Gets a list of installed packages on a device.
  801. Returns:
  802. A list of installed packages on a device.
  803. """
  804. cmd = ['idevicefs', '--udid', self.udid, 'ls', '@']
  805. return print_process_output(self.start_proc(cmd))
  806. def set_up(self):
  807. """Performs setup actions which must occur prior to every test launch."""
  808. self.uninstall_apps()
  809. self.wipe_derived_data()
  810. self.install_app()
  811. def extract_test_data(self):
  812. """Extracts data emitted by the test."""
  813. cmd = [
  814. 'idevicefs',
  815. '--udid', self.udid,
  816. 'pull',
  817. '@%s/Documents' % self.cfbundleid,
  818. os.path.join(self.out_dir, 'Documents'),
  819. ]
  820. try:
  821. print_process_output(self.start_proc(cmd))
  822. except subprocess.CalledProcessError:
  823. raise TestDataExtractionError()
  824. def shutdown_and_restart(self):
  825. """Restart the device, wait for two minutes."""
  826. # TODO(crbug.com/760399): swarming bot ios 11 devices turn to be unavailable
  827. # in a few hours unexpectedly, which is assumed as an ios beta issue. Should
  828. # remove this method once the bug is fixed.
  829. if self.restart:
  830. LOGGER.info('Restarting device, wait for two minutes.')
  831. try:
  832. subprocess.check_call(
  833. ['idevicediagnostics', 'restart', '--udid', self.udid])
  834. except subprocess.CalledProcessError:
  835. raise DeviceRestartError()
  836. time.sleep(120)
  837. def retrieve_crash_reports(self):
  838. """Retrieves crash reports produced by the test."""
  839. logs_dir = os.path.join(self.out_dir, 'Logs')
  840. os.mkdir(logs_dir)
  841. cmd = [
  842. 'idevicecrashreport',
  843. '--extract',
  844. '--udid', self.udid,
  845. logs_dir,
  846. ]
  847. try:
  848. print_process_output(self.start_proc(cmd))
  849. except subprocess.CalledProcessError:
  850. # TODO(crbug.com/828951): Raise the exception when the bug is fixed.
  851. LOGGER.warning('Failed to retrieve crash reports from device.')
  852. def tear_down(self):
  853. """Performs cleanup actions which must occur after every test launch."""
  854. self.retrieve_derived_data()
  855. self.extract_test_data()
  856. self.process_xcresult_dir()
  857. self.retrieve_crash_reports()
  858. self.uninstall_apps()
  859. def get_launch_command(self, test_app, out_dir, destination, shards=1):
  860. """Returns the command that can be used to launch the test app.
  861. Args:
  862. test_app: An app that stores data about test required to run.
  863. out_dir: (str) A path for results.
  864. destination: (str) A destination of device/simulator.
  865. shards: (int) How many shards the tests should be divided into.
  866. Returns:
  867. A list of strings forming the command to launch the test.
  868. """
  869. if self.xctest:
  870. return test_app.command(out_dir, destination, shards)
  871. cmd = [
  872. 'idevice-app-runner',
  873. '--udid', self.udid,
  874. '--start', self.cfbundleid,
  875. ]
  876. args = []
  877. if test_app.included_tests or test_app.excluded_tests:
  878. gtest_filter = test_apps.get_gtest_filter(test_app.included_tests,
  879. test_app.excluded_tests)
  880. args.append('--gtest_filter=%s' % gtest_filter)
  881. for env_var in self.env_vars:
  882. cmd.extend(['-D', env_var])
  883. if args or self.test_args:
  884. cmd.append('--args')
  885. cmd.extend(self.test_args)
  886. cmd.extend(args)
  887. return cmd
  888. def get_launch_env(self):
  889. """Returns a dict of environment variables to use to launch the test app.
  890. Returns:
  891. A dict of environment variables.
  892. """
  893. env = super(DeviceTestRunner, self).get_launch_env()
  894. if self.xctest:
  895. env['NSUnbufferedIO'] = 'YES'
  896. # e.g. ios_web_shell_egtests
  897. env['APP_TARGET_NAME'] = os.path.splitext(
  898. os.path.basename(self.app_path))[0]
  899. # e.g. ios_web_shell_egtests_module
  900. env['TEST_TARGET_NAME'] = env['APP_TARGET_NAME'] + '_module'
  901. return env
  902. def get_launch_test_app(self):
  903. """Returns the proper test_app for the run.
  904. Returns:
  905. A DeviceXCTestUnitTestsApp for the current run to execute.
  906. """
  907. # Non iOS Chrome users have unit tests not built with XCTest.
  908. if not self.xctest:
  909. return test_apps.GTestsApp(
  910. self.app_path,
  911. included_tests=self.test_cases,
  912. env_vars=self.env_vars,
  913. repeat_count=self.repeat_count,
  914. test_args=self.test_args)
  915. return test_apps.DeviceXCTestUnitTestsApp(
  916. self.app_path,
  917. included_tests=self.test_cases,
  918. env_vars=self.env_vars,
  919. repeat_count=self.repeat_count,
  920. test_args=self.test_args)