test_runner_test.py 11 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313
  1. #!/usr/bin/env vpython3
  2. # Copyright 2016 The Chromium Authors. All rights reserved.
  3. # Use of this source code is governed by a BSD-style license that can be
  4. # found in the LICENSE file.
  5. """Unittests for test_runner.py."""
  6. import collections
  7. import logging
  8. import mock
  9. import os
  10. import tempfile
  11. import unittest
  12. import iossim_util
  13. import result_sink_util
  14. import test_apps
  15. from test_result_util import ResultCollection, TestResult, TestStatus
  16. import test_runner
  17. class TestCase(unittest.TestCase):
  18. """Test case which supports installing mocks. Uninstalls on tear down."""
  19. def __init__(self, *args, **kwargs):
  20. """Initializes a new instance of this class."""
  21. super(TestCase, self).__init__(*args, **kwargs)
  22. # Maps object to a dict which maps names of mocked members to their
  23. # original values.
  24. self._mocks = collections.OrderedDict()
  25. def mock(self, obj, member, mock):
  26. """Installs mock in place of the named member of the given obj.
  27. Args:
  28. obj: Any object.
  29. member: String naming the attribute of the object to mock.
  30. mock: The mock to install.
  31. """
  32. self._mocks.setdefault(obj, collections.OrderedDict()).setdefault(
  33. member, getattr(obj, member))
  34. setattr(obj, member, mock)
  35. def unmock(self, obj, member):
  36. """Uninstalls the mock from the named member of given obj.
  37. Args:
  38. obj: An obj who's member has been mocked
  39. member: String naming the attribute of the object to unmock
  40. """
  41. if self._mocks[obj][member]:
  42. setattr(obj, member, self._mocks[obj][member])
  43. def tearDown(self, *args, **kwargs):
  44. """Uninstalls mocks."""
  45. super(TestCase, self).tearDown(*args, **kwargs)
  46. for obj in self._mocks:
  47. for member, original_value in self._mocks[obj].items():
  48. setattr(obj, member, original_value)
  49. class SimulatorTestRunnerTest(TestCase):
  50. """Tests for test_runner.SimulatorTestRunner."""
  51. def setUp(self):
  52. super(SimulatorTestRunnerTest, self).setUp()
  53. self.mock(iossim_util, 'get_simulator', lambda _1, _2: 'sim-UUID')
  54. self.mock(result_sink_util.ResultSinkClient,
  55. 'post', lambda *args, **kwargs: None)
  56. self.mock(test_runner, 'get_current_xcode_info', lambda: {
  57. 'version': 'test version', 'build': 'test build', 'path': 'test/path'})
  58. self.mock(test_apps, 'get_bundle_id', lambda _: 'fake-bundle-id')
  59. self.mock(test_apps.SimulatorXCTestUnitTestsApp,
  60. '_xctest_path', lambda _: 'fake-path')
  61. self.mock(test_apps.plistlib, 'writePlist', lambda _1, _2: '')
  62. self.mock(os.path, 'abspath', lambda path: '/abs/path/to/%s' % path)
  63. self.mock(os.path, 'exists', lambda _: True)
  64. self.mock(test_runner.TestRunner, 'set_sigterm_handler',
  65. lambda self, handler: 0)
  66. self.mock(os, 'listdir', lambda _: [])
  67. def test_app_not_found(self):
  68. """Ensures AppNotFoundError is raised."""
  69. self.mock(os.path, 'exists', lambda p: not p.endswith('fake-app'))
  70. with self.assertRaises(test_runner.AppNotFoundError):
  71. test_runner.SimulatorTestRunner(
  72. 'fake-app',
  73. 'fake-iossim',
  74. 'platform',
  75. 'os',
  76. 'out-dir',
  77. )
  78. def test_iossim_not_found(self):
  79. """Ensures SimulatorNotFoundError is raised."""
  80. self.mock(os.path, 'exists', lambda p: not p.endswith('fake-iossim'))
  81. with self.assertRaises(test_runner.SimulatorNotFoundError):
  82. test_runner.SimulatorTestRunner(
  83. 'fake-app',
  84. 'fake-iossim',
  85. 'iPhone X',
  86. '11.4',
  87. 'out-dir',
  88. )
  89. def test_init(self):
  90. """Ensures instance is created."""
  91. tr = test_runner.SimulatorTestRunner(
  92. 'fake-app',
  93. 'fake-iossim',
  94. 'iPhone X',
  95. '11.4',
  96. 'out-dir',
  97. )
  98. self.assertTrue(tr)
  99. @mock.patch('test_runner.SimulatorTestRunner.tear_down')
  100. @mock.patch('test_runner.SimulatorTestRunner.set_up')
  101. @mock.patch('test_runner.TestRunner._run')
  102. def test_startup_crash(self, mock_run, _1, _2):
  103. """Ensures test is relaunched once on startup crash."""
  104. result = ResultCollection()
  105. result.crashed = True
  106. mock_run.return_value = result
  107. tr = test_runner.SimulatorTestRunner(
  108. 'fake-app',
  109. 'fake-iossim',
  110. 'iPhone X',
  111. '11.4',
  112. 'out-dir',
  113. xctest=True,
  114. )
  115. with self.assertRaises(test_runner.AppLaunchError):
  116. tr.launch()
  117. self.assertEquals(len(mock_run.mock_calls), 2)
  118. def test_relaunch(self):
  119. """Ensures test is relaunched on test crash until tests complete."""
  120. def set_up(self):
  121. return
  122. @staticmethod
  123. def _run(cmd, shards=None):
  124. if not any('retry_after_crash' in cmd_arg for cmd_arg in cmd):
  125. # First run, has no test filter supplied. Mock a crash.
  126. result = ResultCollection(
  127. test_results=[TestResult('crash', TestStatus.CRASH)])
  128. result.crashed = True
  129. result.add_test_result(TestResult('pass', TestStatus.PASS))
  130. result.add_test_result(
  131. TestResult('fail', TestStatus.FAIL, test_log='some logs'))
  132. return result
  133. else:
  134. return ResultCollection(
  135. test_results=[TestResult('crash', TestStatus.PASS)])
  136. def tear_down(self):
  137. return
  138. self.mock(test_runner.SimulatorTestRunner, 'set_up', set_up)
  139. self.mock(test_runner.TestRunner, '_run', _run)
  140. self.mock(test_runner.SimulatorTestRunner, 'tear_down', tear_down)
  141. tr = test_runner.SimulatorTestRunner(
  142. 'fake-app',
  143. 'fake-iossim',
  144. 'iPhone X',
  145. '11.4',
  146. 'out-dir',
  147. )
  148. tr.launch()
  149. self.assertTrue(tr.logs)
  150. @mock.patch('test_runner.SimulatorTestRunner.tear_down')
  151. @mock.patch('test_runner.SimulatorTestRunner.set_up')
  152. @mock.patch('test_runner.TestRunner._run')
  153. def test_failed_test_retry(self, mock_run, _1, _2):
  154. test1_fail_result = TestResult('test1', TestStatus.FAIL)
  155. test2_fail_result = TestResult('test2', TestStatus.FAIL)
  156. test1_pass_result = TestResult('test1', TestStatus.PASS)
  157. test2_pass_result = TestResult('test2', TestStatus.PASS)
  158. result1 = ResultCollection(
  159. test_results=[test1_fail_result, test2_fail_result])
  160. retry_result1 = ResultCollection(test_results=[test1_pass_result])
  161. retry_result2 = ResultCollection(test_results=[test2_pass_result])
  162. mock_run.side_effect = [result1, retry_result1, retry_result2]
  163. tr = test_runner.SimulatorTestRunner(
  164. 'fake-app', 'fake-iossim', 'iPhone X', '11.4', 'out-dir', retries=3)
  165. tr.launch()
  166. self.assertEquals(len(mock_run.mock_calls), 3)
  167. self.assertTrue(tr.logs)
  168. @mock.patch('test_runner.SimulatorTestRunner.tear_down')
  169. @mock.patch('test_runner.SimulatorTestRunner.set_up')
  170. @mock.patch('test_runner.TestRunner._run')
  171. def test_crashed_if_crash_in_final_crash_retry(self, mock_run, _1, _2):
  172. test1_crash_result = TestResult('test1', TestStatus.CRASH)
  173. test2_crash_result = TestResult('test2', TestStatus.CRASH)
  174. test3_pass_result = TestResult('test3', TestStatus.PASS)
  175. test1_pass_result = TestResult('test1', TestStatus.PASS)
  176. test2_pass_result = TestResult('test2', TestStatus.PASS)
  177. initial_result = ResultCollection(test_results=[test1_crash_result])
  178. initial_result.crashed = True
  179. crash_retry1_result = ResultCollection(test_results=[test2_crash_result])
  180. crash_retry1_result.crashed = True
  181. crash_retry2_result = ResultCollection(test_results=[test3_pass_result])
  182. crash_retry2_result.crashed = True
  183. test_retry1_result = ResultCollection(test_results=[test1_pass_result])
  184. test_retry2_result = ResultCollection(test_results=[test2_pass_result])
  185. mock_run.side_effect = [
  186. initial_result, crash_retry1_result, crash_retry2_result,
  187. test_retry1_result, test_retry2_result
  188. ]
  189. tr = test_runner.SimulatorTestRunner(
  190. 'fake-app', 'fake-iossim', 'iPhone X', '11.4', 'out-dir', retries=3)
  191. tr.launch()
  192. self.assertEquals(len(mock_run.mock_calls), 5)
  193. self.assertTrue(tr.test_results['interrupted'])
  194. self.assertIn('test suite crash', tr.logs)
  195. self.assertTrue(tr.logs)
  196. @mock.patch('test_runner.SimulatorTestRunner.tear_down')
  197. @mock.patch('test_runner.SimulatorTestRunner.set_up')
  198. @mock.patch('test_runner.TestRunner._run')
  199. def test_not_crashed_if_no_crash_in_final_crash_retry(self, mock_run, _1, _2):
  200. test1_crash_result = TestResult('test1', TestStatus.CRASH)
  201. test2_crash_result = TestResult('test2', TestStatus.CRASH)
  202. test3_pass_result = TestResult('test3', TestStatus.PASS)
  203. test1_pass_result = TestResult('test1', TestStatus.PASS)
  204. test2_pass_result = TestResult('test2', TestStatus.PASS)
  205. initial_result = ResultCollection(test_results=[test1_crash_result])
  206. initial_result.crashed = True
  207. crash_retry1_result = ResultCollection(test_results=[test2_crash_result])
  208. crash_retry1_result.crashed = True
  209. crash_retry2_result = ResultCollection(test_results=[test3_pass_result])
  210. test_retry1_result = ResultCollection(test_results=[test1_pass_result])
  211. test_retry2_result = ResultCollection(test_results=[test2_pass_result])
  212. mock_run.side_effect = [
  213. initial_result, crash_retry1_result, crash_retry2_result,
  214. test_retry1_result, test_retry2_result
  215. ]
  216. tr = test_runner.SimulatorTestRunner(
  217. 'fake-app', 'fake-iossim', 'iPhone X', '11.4', 'out-dir', retries=3)
  218. tr.launch()
  219. self.assertEquals(len(mock_run.mock_calls), 5)
  220. self.assertFalse(tr.test_results['interrupted'])
  221. self.assertTrue(tr.logs)
  222. @mock.patch('test_runner.SimulatorTestRunner.tear_down')
  223. @mock.patch('test_runner.SimulatorTestRunner.set_up')
  224. @mock.patch('test_runner.TestRunner._run')
  225. def test_not_crashed_if_crashed_in_failed_test_retry(self, mock_run, _1, _2):
  226. test1_fail_result = TestResult('test1', TestStatus.FAIL)
  227. initial_result = ResultCollection(test_results=[test1_fail_result])
  228. test1_retry1_result = ResultCollection(test_results=[test1_fail_result])
  229. test1_retry2_result = ResultCollection(test_results=[test1_fail_result])
  230. test1_retry3_result = ResultCollection()
  231. test1_retry3_result.crashed = True
  232. mock_run.side_effect = [
  233. initial_result, test1_retry1_result, test1_retry2_result,
  234. test1_retry3_result
  235. ]
  236. tr = test_runner.SimulatorTestRunner(
  237. 'fake-app', 'fake-iossim', 'iPhone X', '11.4', 'out-dir', retries=3)
  238. tr.launch()
  239. self.assertEquals(len(mock_run.mock_calls), 4)
  240. self.assertFalse(tr.test_results['interrupted'])
  241. self.assertEquals(tr.test_results['tests']['test1']['actual'],
  242. 'FAIL FAIL FAIL SKIP')
  243. self.assertTrue(tr.logs)
  244. class DeviceTestRunnerTest(TestCase):
  245. def setUp(self):
  246. super(DeviceTestRunnerTest, self).setUp()
  247. def install_xcode(build, mac_toolchain_cmd, xcode_app_path):
  248. return True
  249. self.mock(result_sink_util.ResultSinkClient,
  250. 'post', lambda *args, **kwargs: None)
  251. self.mock(test_runner, 'get_current_xcode_info', lambda: {
  252. 'version': 'test version', 'build': 'test build', 'path': 'test/path'})
  253. self.mock(test_runner, 'install_xcode', install_xcode)
  254. self.mock(test_runner.subprocess,
  255. 'check_output', lambda _: b'fake-bundle-id')
  256. self.mock(os.path, 'abspath', lambda path: '/abs/path/to/%s' % path)
  257. self.mock(os.path, 'exists', lambda _: True)
  258. self.mock(os, 'listdir', lambda _: [])
  259. self.mock(tempfile, 'mkstemp', lambda: '/tmp/tmp_file')
  260. self.tr = test_runner.DeviceTestRunner(
  261. 'fake-app',
  262. 'xcode-version',
  263. 'xcode-build',
  264. 'out-dir',
  265. )
  266. self.tr.xctestrun_data = {'TestTargetName': {}}
  267. if __name__ == '__main__':
  268. logging.basicConfig(format='[%(asctime)s:%(levelname)s] %(message)s',
  269. level=logging.DEBUG, datefmt='%I:%M:%S')
  270. unittest.main()