xcodebuild_runner.py 16 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414
  1. # Copyright 2018 The Chromium Authors. All rights reserved.
  2. # Use of this source code is governed by a BSD-style license that can be
  3. # found in the LICENSE file.
  4. """Test runner for running tests using xcodebuild."""
  5. import collections
  6. import logging
  7. import os
  8. import subprocess
  9. import time
  10. import file_util
  11. import iossim_util
  12. import test_apps
  13. from test_result_util import ResultCollection, TestResult, TestStatus
  14. import test_runner
  15. import xcode_log_parser
  16. LOGGER = logging.getLogger(__name__)
  17. MAXIMUM_TESTS_PER_SHARD_FOR_RERUN = 20
  18. XTDEVICE_FOLDER = os.path.expanduser('~/Library/Developer/XCTestDevices')
  19. def _tests_decided_at_runtime(app_name):
  20. """Return if tests in app are selected at runtime by app_name.
  21. This works for suites defined in chromium infra.
  22. """
  23. suite_name_fragments = ['ios_chrome_multitasking_eg', '_flaky_eg']
  24. return any(fragment in app_name for fragment in suite_name_fragments)
  25. def erase_all_simulators(path=None):
  26. """Erases all simulator devices.
  27. Args:
  28. path: (str) A path with simulators
  29. """
  30. command = ['xcrun', 'simctl']
  31. if path:
  32. command += ['--set', path]
  33. LOGGER.info('Erasing all simulators from folder %s.' % path)
  34. else:
  35. LOGGER.info('Erasing all simulators.')
  36. try:
  37. subprocess.check_call(command + ['erase', 'all'])
  38. except subprocess.CalledProcessError as e:
  39. # Logging error instead of throwing so we don't cause failures in case
  40. # this was indeed failing to clean up.
  41. message = 'Failed to erase all simulators. Error: %s' % e.output
  42. LOGGER.error(message)
  43. def shutdown_all_simulators(path=None):
  44. """Shutdown all simulator devices.
  45. Fix for DVTCoreSimulatorAdditionsErrorDomain error.
  46. Args:
  47. path: (str) A path with simulators
  48. """
  49. command = ['xcrun', 'simctl']
  50. if path:
  51. command += ['--set', path]
  52. LOGGER.info('Shutdown all simulators from folder %s.' % path)
  53. else:
  54. LOGGER.info('Shutdown all simulators.')
  55. try:
  56. subprocess.check_call(command + ['shutdown', 'all'])
  57. except subprocess.CalledProcessError as e:
  58. # Logging error instead of throwing so we don't cause failures in case
  59. # this was indeed failing to clean up.
  60. message = 'Failed to shutdown all simulators. Error: %s' % e.output
  61. LOGGER.error(message)
  62. def terminate_process(proc):
  63. """Terminates the process.
  64. If an error occurs ignore it, just print out a message.
  65. Args:
  66. proc: A subprocess.
  67. """
  68. try:
  69. proc.terminate()
  70. except OSError as ex:
  71. LOGGER.error('Error while killing a process: %s' % ex)
  72. class LaunchCommand(object):
  73. """Stores xcodebuild test launching command."""
  74. def __init__(self,
  75. egtests_app,
  76. udid,
  77. shards,
  78. retries,
  79. readline_timeout,
  80. out_dir=os.path.basename(os.getcwd()),
  81. use_clang_coverage=False,
  82. env=None):
  83. """Initialize launch command.
  84. Args:
  85. egtests_app: (EgtestsApp) An egtests_app to run.
  86. udid: (str) UDID of a device/simulator.
  87. shards: (int) A number of shards.
  88. readline_timeout: (int) Timeout to kill a test process when it doesn't
  89. have output (in seconds).
  90. retries: (int) A number of retries.
  91. out_dir: (str) A folder in which xcodebuild will generate test output.
  92. By default it is a current directory.
  93. env: (dict) Environment variables.
  94. Raises:
  95. AppNotFoundError: At incorrect egtests_app parameter type.
  96. """
  97. if not isinstance(egtests_app, test_apps.EgtestsApp):
  98. raise test_runner.AppNotFoundError(
  99. 'Parameter `egtests_app` is not EgtestsApp: %s' % egtests_app)
  100. self.egtests_app = egtests_app
  101. self.udid = udid
  102. self.shards = shards
  103. self.readline_timeout = readline_timeout
  104. self.retries = retries
  105. self.out_dir = out_dir
  106. self.use_clang_coverage = use_clang_coverage
  107. self.env = env
  108. self._log_parser = xcode_log_parser.get_parser()
  109. def launch_attempt(self, cmd):
  110. """Launch a process and do logging simultaneously.
  111. Args:
  112. cmd: (list[str]) A command to run.
  113. Returns:
  114. output - command output as list of strings.
  115. """
  116. proc = subprocess.Popen(
  117. cmd,
  118. env=self.env,
  119. stdout=subprocess.PIPE,
  120. stderr=subprocess.STDOUT,
  121. )
  122. return test_runner.print_process_output(proc, timeout=self.readline_timeout)
  123. def launch(self):
  124. """Launches tests using xcodebuild."""
  125. overall_launch_command_result = ResultCollection()
  126. shards = self.shards
  127. running_tests = set(self.egtests_app.get_all_tests())
  128. # total number of attempts is self.retries+1
  129. for attempt in range(self.retries + 1):
  130. # Erase all simulators per each attempt
  131. if iossim_util.is_device_with_udid_simulator(self.udid):
  132. # kill all running simulators to prevent possible memory leaks
  133. test_runner.SimulatorTestRunner.kill_simulators()
  134. shutdown_all_simulators()
  135. shutdown_all_simulators(XTDEVICE_FOLDER)
  136. erase_all_simulators()
  137. erase_all_simulators(XTDEVICE_FOLDER)
  138. outdir_attempt = os.path.join(self.out_dir, 'attempt_%d' % attempt)
  139. cmd_list = self.egtests_app.command(outdir_attempt, 'id=%s' % self.udid,
  140. shards)
  141. # TODO(crbug.com/914878): add heartbeat logging to xcodebuild_runner.
  142. LOGGER.info('Start test attempt #%d for command [%s]' % (
  143. attempt, ' '.join(cmd_list)))
  144. output = self.launch_attempt(cmd_list)
  145. if hasattr(self, 'use_clang_coverage') and self.use_clang_coverage:
  146. # out_dir of LaunchCommand object is the TestRunner out_dir joined with
  147. # UDID. Use os.path.dirname to retrieve the TestRunner out_dir.
  148. file_util.move_raw_coverage_data(self.udid,
  149. os.path.dirname(self.out_dir))
  150. result = self._log_parser.collect_test_results(outdir_attempt, output)
  151. tests_selected_at_runtime = _tests_decided_at_runtime(
  152. self.egtests_app.test_app_path)
  153. # For most suites, only keep crash status from last attempt since retries
  154. # will cover any missing tests. For these decided at runtime, retain
  155. # crashes from all attempts and a dummy "crashed" result will be reported
  156. # to indicate some tests might never ran.
  157. # TODO(crbug.com/1235871): Switch back to excluded tests and set
  158. # |overall_crash| to always True.
  159. overall_launch_command_result.add_result_collection(
  160. result, overwrite_crash=not tests_selected_at_runtime)
  161. result.report_to_result_sink()
  162. tests_to_include = set()
  163. # |running_tests| are compiled tests in target intersecting with swarming
  164. # sharding. For some suites, they are more than what's needed to run.
  165. if not tests_selected_at_runtime:
  166. tests_to_include = tests_to_include | (
  167. running_tests - overall_launch_command_result.expected_tests())
  168. # Add failed tests from last rounds for runtime decided suites and device
  169. # suites.
  170. tests_to_include = (
  171. tests_to_include
  172. | overall_launch_command_result.never_expected_tests())
  173. self.egtests_app.included_tests = list(tests_to_include)
  174. # Nothing to run in retry.
  175. if not self.egtests_app.included_tests:
  176. break
  177. # If tests are not completed(interrupted or did not start) and there are
  178. # >= 20 remaining tests, run them with the same number of shards.
  179. # otherwise re-run with shards=1.
  180. if (not result.crashed
  181. # If need to re-run less than 20 tests, 1 shard should be enough.
  182. or (len(running_tests) -
  183. len(overall_launch_command_result.expected_tests()) <=
  184. MAXIMUM_TESTS_PER_SHARD_FOR_RERUN)):
  185. shards = 1
  186. return overall_launch_command_result
  187. class SimulatorParallelTestRunner(test_runner.SimulatorTestRunner):
  188. """Class for running simulator tests using xCode."""
  189. def __init__(self, app_path, host_app_path, iossim_path, version, platform,
  190. out_dir, **kwargs):
  191. """Initializes a new instance of SimulatorParallelTestRunner class.
  192. Args:
  193. app_path: (str) A path to egtests_app.
  194. host_app_path: (str) A path to the host app for EG2.
  195. iossim_path: Path to the compiled iossim binary to use.
  196. Not used, but is required by the base class.
  197. version: (str) iOS version to run simulator on.
  198. platform: (str) Name of device.
  199. out_dir: (str) A directory to emit test data into.
  200. (Following are potential args in **kwargs)
  201. release: (bool) Whether this test runner is running for a release build.
  202. repeat_count: (int) Number of times to run each test (passed to test app).
  203. retries: (int) A number to retry test run, will re-run only failed tests.
  204. shards: (int) A number of shards. Default is 1.
  205. test_cases: (list) List of tests to be included in the test run.
  206. None or [] to include all tests.
  207. test_args: List of strings to pass as arguments to the test when
  208. launching.
  209. use_clang_coverage: Whether code coverage is enabled in this run.
  210. env_vars: List of environment variables to pass to the test itself.
  211. Raises:
  212. AppNotFoundError: If the given app does not exist.
  213. PlugInsNotFoundError: If the PlugIns directory does not exist for XCTests.
  214. XcodeVersionNotFoundError: If the given Xcode version does not exist.
  215. XCTestPlugInNotFoundError: If the .xctest PlugIn does not exist.
  216. """
  217. kwargs['retries'] = kwargs.get('retries') or 0
  218. super(SimulatorParallelTestRunner,
  219. self).__init__(app_path, iossim_path, platform, version, out_dir,
  220. **kwargs)
  221. self.set_up()
  222. self.host_app_path = None
  223. if host_app_path != 'NO_PATH':
  224. self.host_app_path = os.path.abspath(host_app_path)
  225. self.logs = collections.OrderedDict()
  226. self.release = kwargs.get('release') or False
  227. self.test_results['path_delimiter'] = '/'
  228. # Do not enable parallel testing when code coverage is enabled, because raw
  229. # coverage data won't be produced with parallel testing.
  230. if hasattr(self, 'use_clang_coverage') and self.use_clang_coverage:
  231. self.shards = 1
  232. def get_launch_env(self):
  233. """Returns a dict of environment variables to use to launch the test app.
  234. Returns:
  235. A dict of environment variables.
  236. """
  237. env = super(test_runner.SimulatorTestRunner, self).get_launch_env()
  238. env['NSUnbufferedIO'] = 'YES'
  239. return env
  240. def get_launch_test_app(self):
  241. """Returns the proper test_app for the run.
  242. Returns:
  243. An implementation of EgtestsApp for the runner.
  244. """
  245. return test_apps.EgtestsApp(
  246. self.app_path,
  247. included_tests=self.test_cases,
  248. env_vars=self.env_vars,
  249. test_args=self.test_args,
  250. release=self.release,
  251. repeat_count=self.repeat_count,
  252. host_app_path=self.host_app_path)
  253. def launch(self):
  254. """Launches tests using xcodebuild."""
  255. test_app = self.get_launch_test_app()
  256. launch_command = LaunchCommand(
  257. test_app,
  258. udid=self.udid,
  259. shards=self.shards,
  260. readline_timeout=self.readline_timeout,
  261. retries=self.retries,
  262. out_dir=os.path.join(self.out_dir, self.udid),
  263. use_clang_coverage=(hasattr(self, 'use_clang_coverage') and
  264. self.use_clang_coverage),
  265. env=self.get_launch_env())
  266. overall_result = launch_command.launch()
  267. # Deletes simulator used in the tests after tests end.
  268. if iossim_util.is_device_with_udid_simulator(self.udid):
  269. iossim_util.delete_simulator_by_udid(self.udid)
  270. # Adds disabled tests to result.
  271. overall_result.add_and_report_test_names_status(
  272. launch_command.egtests_app.disabled_tests,
  273. TestStatus.SKIP,
  274. expected_status=TestStatus.SKIP,
  275. test_log='Test disabled.')
  276. # Adds unexpectedly skipped tests to result if applicable.
  277. tests_selected_at_runtime = _tests_decided_at_runtime(self.app_path)
  278. unexpectedly_skipped = []
  279. # TODO(crbug.com/1048758): For the multitasking or any flaky test suites,
  280. # |all_tests_to_run| contains more tests than what actually runs.
  281. if not tests_selected_at_runtime:
  282. # |all_tests_to_run| takes into consideration that only a subset of tests
  283. # may have run due to the test sharding logic in run.py.
  284. all_tests_to_run = set(launch_command.egtests_app.get_all_tests())
  285. unexpectedly_skipped = list(all_tests_to_run -
  286. overall_result.all_test_names())
  287. overall_result.add_and_report_test_names_status(
  288. unexpectedly_skipped,
  289. TestStatus.SKIP,
  290. test_log=('The test is compiled in test target but was unexpectedly '
  291. 'not run or not finished.'))
  292. # Add a final crash status to result collection. It will be reported as
  293. # part of step log in LUCI build.
  294. if unexpectedly_skipped or overall_result.crashed:
  295. overall_result.set_crashed_with_prefix(
  296. crash_message_prefix_line=('Test application crash happened and may '
  297. 'result in missing tests:'))
  298. self.test_results = overall_result.standard_json_output(path_delimiter='/')
  299. self.logs.update(overall_result.test_runner_logs())
  300. # Return False when:
  301. # - There are unexpected tests (all results of the tests are unexpected), or
  302. # - The overall status is crashed and tests are selected at runtime. (i.e.
  303. # runner is unable to know if all scheduled tests appear in result.)
  304. return (not overall_result.never_expected_tests() and
  305. not (tests_selected_at_runtime and overall_result.crashed))
  306. class DeviceXcodeTestRunner(SimulatorParallelTestRunner,
  307. test_runner.DeviceTestRunner):
  308. """Class for running tests on real device using xCode."""
  309. def __init__(self, app_path, host_app_path, out_dir, **kwargs):
  310. """Initializes a new instance of DeviceXcodeTestRunner class.
  311. Args:
  312. app_path: (str) A path to egtests_app.
  313. host_app_path: (str) A path to the host app for EG2.
  314. out_dir: (str) A directory to emit test data into.
  315. (Following are potential args in **kwargs)
  316. repeat_count: (int) Number of times to run each test (passed to test app).
  317. retries: (int) A number to retry test run, will re-run only failed tests.
  318. test_cases: (list) List of tests to be included in the test run.
  319. None or [] to include all tests.
  320. test_args: List of strings to pass as arguments to the test when
  321. launching.
  322. env_vars: List of environment variables to pass to the test itself.
  323. Raises:
  324. AppNotFoundError: If the given app does not exist.
  325. DeviceDetectionError: If no device found.
  326. PlugInsNotFoundError: If the PlugIns directory does not exist for XCTests.
  327. XcodeVersionNotFoundError: If the given Xcode version does not exist.
  328. XCTestPlugInNotFoundError: If the .xctest PlugIn does not exist.
  329. """
  330. test_runner.DeviceTestRunner.__init__(self, app_path, out_dir, **kwargs)
  331. self.shards = 1 # For tests on real devices shards=1
  332. self.version = None
  333. self.platform = None
  334. self.host_app_path = None
  335. if host_app_path != 'NO_PATH':
  336. self.host_app_path = os.path.abspath(host_app_path)
  337. self.homedir = ''
  338. self.release = kwargs.get('release') or False
  339. self.set_up()
  340. self.start_time = time.strftime('%Y-%m-%d-%H%M%S', time.localtime())
  341. self.test_results['path_delimiter'] = '/'
  342. def set_up(self):
  343. """Performs setup actions which must occur prior to every test launch."""
  344. self.uninstall_apps()
  345. self.wipe_derived_data()
  346. def tear_down(self):
  347. """Performs cleanup actions which must occur after every test launch."""
  348. test_runner.DeviceTestRunner.tear_down(self)
  349. def launch(self):
  350. try:
  351. return super(DeviceXcodeTestRunner, self).launch()
  352. finally:
  353. self.tear_down()