test_result_util_test.py 17 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429
  1. #!/usr/bin/env vpython3
  2. # Copyright 2021 The Chromium Authors. All rights reserved.
  3. # Use of this source code is governed by a BSD-style license that can be
  4. # found in the LICENSE file.
  5. """Unittests for test_result_util.py."""
  6. import collections
  7. import copy
  8. import mock
  9. import unittest
  10. import test_result_util
  11. from test_result_util import TestResult, TestStatus, ResultCollection
  12. import test_runner_test
  13. PASSED_RESULT = TestResult(
  14. 'passed/test', TestStatus.PASS, duration=1233, test_log='Logs')
  15. FAILED_RESULT = TestResult(
  16. 'failed/test', TestStatus.FAIL, duration=1233, test_log='line1\nline2')
  17. FAILED_RESULT_DUPLICATE = TestResult(
  18. 'failed/test', TestStatus.FAIL, test_log='line3\nline4')
  19. DISABLED_RESULT = TestResult(
  20. 'disabled/test',
  21. TestStatus.SKIP,
  22. expected_status=TestStatus.SKIP,
  23. attachments={'name': '/path/to/name'})
  24. UNEXPECTED_SKIPPED_RESULT = TestResult('unexpected/skipped_test',
  25. TestStatus.SKIP)
  26. CRASHED_RESULT = TestResult('crashed/test', TestStatus.CRASH)
  27. FLAKY_PASS_RESULT = TestResult('flaky/test', TestStatus.PASS)
  28. FLAKY_FAIL_RESULT = TestResult(
  29. 'flaky/test', TestStatus.FAIL, test_log='line1\nline2')
  30. ABORTED_RESULT = TestResult('aborted/test', TestStatus.ABORT)
  31. class UtilTest(test_runner_test.TestCase):
  32. """Tests util methods in test_result_util module."""
  33. def test_validate_kwargs(self):
  34. """Tests _validate_kwargs."""
  35. with self.assertRaises(AssertionError) as context:
  36. TestResult('name', TestStatus.PASS, unknown='foo')
  37. expected_message = ("Invalid keyword argument(s) in")
  38. self.assertTrue(expected_message in str(context.exception))
  39. with self.assertRaises(AssertionError) as context:
  40. ResultCollection(test_log='foo')
  41. expected_message = ("Invalid keyword argument(s) in")
  42. self.assertTrue(expected_message in str(context.exception))
  43. def test_validate_test_status(self):
  44. """Tests exception raised from validation."""
  45. with self.assertRaises(TypeError) as context:
  46. test_result_util._validate_test_status('TIMEOUT')
  47. expected_message = ('Invalid test status: TIMEOUT. Should be one of')
  48. self.assertTrue(expected_message in str(context.exception))
  49. def test_to_standard_json_literal(self):
  50. """Tests _to_standard_json_literal."""
  51. status = test_result_util._to_standard_json_literal(TestStatus.FAIL)
  52. self.assertEqual(status, 'FAIL')
  53. status = test_result_util._to_standard_json_literal(TestStatus.ABORT)
  54. self.assertEqual(status, 'TIMEOUT')
  55. class TestResultTest(test_runner_test.TestCase):
  56. """Tests TestResult class APIs."""
  57. def test_init(self):
  58. """Tests class initialization."""
  59. test_result = PASSED_RESULT
  60. self.assertEqual(test_result.name, 'passed/test')
  61. self.assertEqual(test_result.status, TestStatus.PASS)
  62. self.assertEqual(test_result.expected_status, TestStatus.PASS)
  63. self.assertEqual(test_result.test_log, 'Logs')
  64. def test_compose_result_sink_tags(self):
  65. """Tests _compose_result_sink_tags."""
  66. disabled_test_tags = [('test_name', 'disabled/test'),
  67. ('disabled_test', 'true')]
  68. unexpected_skip_test_tags = [('test_name', 'unexpected/skipped_test'),
  69. ('disabled_test', 'false')]
  70. not_skip_test_tags = [('test_name', 'passed/test')]
  71. not_skip_test_result = PASSED_RESULT
  72. self.assertEqual(not_skip_test_tags,
  73. not_skip_test_result._compose_result_sink_tags())
  74. disabled_test_result = DISABLED_RESULT
  75. self.assertEqual(disabled_test_tags,
  76. disabled_test_result._compose_result_sink_tags())
  77. unexpected_skip_test_result = UNEXPECTED_SKIPPED_RESULT
  78. self.assertEqual(unexpected_skip_test_tags,
  79. unexpected_skip_test_result._compose_result_sink_tags())
  80. @mock.patch('result_sink_util.ResultSinkClient.post')
  81. def test_report_to_result_sink(self, mock_post):
  82. disabled_test_result = DISABLED_RESULT
  83. client = mock.MagicMock()
  84. disabled_test_result.report_to_result_sink(client)
  85. client.post.assert_called_with(
  86. 'disabled/test',
  87. 'SKIP',
  88. True,
  89. duration=None,
  90. test_log='',
  91. tags=[('test_name', 'disabled/test'), ('disabled_test', 'true')],
  92. file_artifacts={'name': '/path/to/name'})
  93. # Duplicate calls will only report once.
  94. disabled_test_result.report_to_result_sink(client)
  95. self.assertEqual(client.post.call_count, 1)
  96. disabled_test_result.report_to_result_sink(client)
  97. self.assertEqual(client.post.call_count, 1)
  98. faileded_result = FAILED_RESULT
  99. client = mock.MagicMock()
  100. faileded_result.report_to_result_sink(client)
  101. client.post.assert_called_with(
  102. 'failed/test',
  103. 'FAIL',
  104. False,
  105. duration=1233,
  106. file_artifacts={},
  107. tags=[('test_name', 'failed/test')],
  108. test_log='line1\nline2')
  109. class ResultCollectionTest(test_runner_test.TestCase):
  110. """Tests ResultCollection class APIs."""
  111. def setUp(self):
  112. super(ResultCollectionTest, self).setUp()
  113. self.full_collection = ResultCollection(test_results=[
  114. PASSED_RESULT, FAILED_RESULT, FAILED_RESULT_DUPLICATE, DISABLED_RESULT,
  115. UNEXPECTED_SKIPPED_RESULT, CRASHED_RESULT, FLAKY_PASS_RESULT,
  116. FLAKY_FAIL_RESULT, ABORTED_RESULT
  117. ])
  118. def test_init(self):
  119. """Tests class initialization."""
  120. collection = ResultCollection(
  121. test_results=[
  122. PASSED_RESULT, DISABLED_RESULT, UNEXPECTED_SKIPPED_RESULT
  123. ],
  124. crashed=True)
  125. self.assertTrue(collection.crashed)
  126. self.assertEqual(collection.crash_message, '')
  127. self.assertEqual(
  128. collection.test_results,
  129. [PASSED_RESULT, DISABLED_RESULT, UNEXPECTED_SKIPPED_RESULT])
  130. def test_add_result(self):
  131. """Tests add_test_result."""
  132. collection = ResultCollection(test_results=[FAILED_RESULT])
  133. collection.add_test_result(DISABLED_RESULT)
  134. self.assertEqual(collection.test_results, [FAILED_RESULT, DISABLED_RESULT])
  135. def test_add_result_collection_default(self):
  136. """Tests add_result_collection default (merge crash info)."""
  137. collection = ResultCollection(test_results=[FAILED_RESULT])
  138. self.assertFalse(collection.crashed)
  139. collection.append_crash_message('Crash1')
  140. crashed_collection = ResultCollection(
  141. test_results=[PASSED_RESULT], crashed=True)
  142. crashed_collection.append_crash_message('Crash2')
  143. collection.add_result_collection(crashed_collection)
  144. self.assertTrue(collection.crashed)
  145. self.assertEqual(collection.crash_message, 'Crash1\nCrash2')
  146. self.assertEqual(collection.test_results, [FAILED_RESULT, PASSED_RESULT])
  147. def test_add_result_collection_overwrite(self):
  148. """Tests add_result_collection overwrite."""
  149. collection = ResultCollection(test_results=[FAILED_RESULT], crashed=True)
  150. self.assertTrue(collection.crashed)
  151. collection.append_crash_message('Crash1')
  152. crashed_collection = ResultCollection(test_results=[PASSED_RESULT])
  153. collection.add_result_collection(crashed_collection, overwrite_crash=True)
  154. self.assertFalse(collection.crashed)
  155. self.assertEqual(collection.crash_message, '')
  156. self.assertEqual(collection.test_results, [FAILED_RESULT, PASSED_RESULT])
  157. def test_add_result_collection_ignore(self):
  158. """Tests add_result_collection overwrite."""
  159. collection = ResultCollection(test_results=[FAILED_RESULT])
  160. self.assertFalse(collection.crashed)
  161. crashed_collection = ResultCollection(
  162. test_results=[PASSED_RESULT], crashed=True)
  163. crashed_collection.append_crash_message('Crash2')
  164. collection.add_result_collection(crashed_collection, ignore_crash=True)
  165. self.assertFalse(collection.crashed)
  166. self.assertEqual(collection.crash_message, '')
  167. self.assertEqual(collection.test_results, [FAILED_RESULT, PASSED_RESULT])
  168. def test_add_results(self):
  169. """Tests add_results."""
  170. collection = ResultCollection(test_results=[PASSED_RESULT])
  171. collection.add_results([FAILED_RESULT, DISABLED_RESULT])
  172. self.assertEqual(collection.test_results,
  173. [PASSED_RESULT, FAILED_RESULT, DISABLED_RESULT])
  174. def test_add_name_prefix_to_tests(self):
  175. """Tests add_name_prefix_to_tests."""
  176. passed = copy.copy(PASSED_RESULT)
  177. disabeld = copy.copy(DISABLED_RESULT)
  178. collection = ResultCollection(test_results=[passed, disabeld])
  179. some_prefix = 'Some/prefix'
  180. collection.add_name_prefix_to_tests(some_prefix)
  181. for test_result in collection.test_results:
  182. self.assertTrue(test_result.name.startswith(some_prefix))
  183. def test_add_test_names_status(self):
  184. """Tests add_test_names_status."""
  185. test_names = ['test1', 'test2', 'test3']
  186. collection = ResultCollection(test_results=[PASSED_RESULT])
  187. collection.add_test_names_status(test_names, TestStatus.SKIP)
  188. disabled_test_names = ['test4', 'test5', 'test6']
  189. collection.add_test_names_status(
  190. disabled_test_names, TestStatus.SKIP, expected_status=TestStatus.SKIP)
  191. self.assertEqual(collection.test_results[0], PASSED_RESULT)
  192. unexpected_skipped = collection.tests_by_expression(
  193. lambda t: not t.expected() and t.status == TestStatus.SKIP)
  194. self.assertEqual(unexpected_skipped, set(['test1', 'test2', 'test3']))
  195. self.assertEqual(collection.disabled_tests(),
  196. set(['test4', 'test5', 'test6']))
  197. @mock.patch('test_result_util.TestResult.report_to_result_sink')
  198. @mock.patch('result_sink_util.ResultSinkClient.close')
  199. @mock.patch('result_sink_util.ResultSinkClient.__init__', return_value=None)
  200. def test_add_and_report_test_names_status(self, mock_sink_init,
  201. mock_sink_close, mock_report):
  202. """Tests add_test_names_status."""
  203. test_names = ['test1', 'test2', 'test3']
  204. collection = ResultCollection(test_results=[PASSED_RESULT])
  205. collection.add_and_report_test_names_status(test_names, TestStatus.SKIP)
  206. self.assertEqual(collection.test_results[0], PASSED_RESULT)
  207. unexpected_skipped = collection.tests_by_expression(
  208. lambda t: not t.expected() and t.status == TestStatus.SKIP)
  209. self.assertEqual(unexpected_skipped, set(['test1', 'test2', 'test3']))
  210. self.assertEqual(1, len(mock_sink_init.mock_calls))
  211. self.assertEqual(3, len(mock_report.mock_calls))
  212. self.assertEqual(1, len(mock_sink_close.mock_calls))
  213. def testappend_crash_message(self):
  214. """Tests append_crash_message."""
  215. collection = ResultCollection(test_results=[PASSED_RESULT])
  216. collection.append_crash_message('Crash message 1.')
  217. self.assertEqual(collection.crash_message, 'Crash message 1.')
  218. collection.append_crash_message('Crash message 2.')
  219. self.assertEqual(collection.crash_message,
  220. 'Crash message 1.\nCrash message 2.')
  221. def test_tests_by_expression(self):
  222. """Tests tests_by_expression."""
  223. collection = self.full_collection
  224. exp = lambda result: result.status == TestStatus.SKIP
  225. skipped_tests = collection.tests_by_expression(exp)
  226. self.assertEqual(skipped_tests,
  227. set(['unexpected/skipped_test', 'disabled/test']))
  228. def test_get_spcific_tests(self):
  229. """Tests getting sets of tests of specific status."""
  230. collection = self.full_collection
  231. self.assertEqual(
  232. collection.all_test_names(),
  233. set([
  234. 'passed/test', 'disabled/test', 'failed/test',
  235. 'unexpected/skipped_test', 'crashed/test', 'flaky/test',
  236. 'aborted/test'
  237. ]))
  238. self.assertEqual(collection.crashed_tests(), set(['crashed/test']))
  239. self.assertEqual(collection.disabled_tests(), set(['disabled/test']))
  240. self.assertEqual(collection.expected_tests(),
  241. set(['passed/test', 'disabled/test', 'flaky/test']))
  242. self.assertEqual(
  243. collection.unexpected_tests(),
  244. set([
  245. 'failed/test', 'unexpected/skipped_test', 'crashed/test',
  246. 'flaky/test', 'aborted/test'
  247. ]))
  248. self.assertEqual(collection.passed_tests(),
  249. set(['passed/test', 'flaky/test']))
  250. self.assertEqual(collection.failed_tests(),
  251. set(['failed/test', 'flaky/test']))
  252. self.assertEqual(collection.flaky_tests(), set(['flaky/test']))
  253. self.assertEqual(
  254. collection.never_expected_tests(),
  255. set([
  256. 'failed/test', 'unexpected/skipped_test', 'crashed/test',
  257. 'aborted/test'
  258. ]))
  259. self.assertEqual(collection.pure_expected_tests(),
  260. set(['passed/test', 'disabled/test']))
  261. def test_add_and_report_crash(self):
  262. """Tests add_and_report_crash."""
  263. collection = copy.copy(self.full_collection)
  264. collection.set_crashed_with_prefix('Prefix Line')
  265. self.assertEqual(collection.crash_message, 'Prefix Line\n')
  266. self.assertTrue(collection.crashed)
  267. @mock.patch('test_result_util.TestResult.report_to_result_sink')
  268. @mock.patch('result_sink_util.ResultSinkClient.close')
  269. @mock.patch('result_sink_util.ResultSinkClient.__init__', return_value=None)
  270. def test_report_to_result_sink(self, mock_sink_init, mock_sink_close,
  271. mock_report):
  272. """Tests report_to_result_sink."""
  273. collection = copy.copy(self.full_collection)
  274. collection.report_to_result_sink()
  275. mock_sink_init.assert_called_once()
  276. self.assertEqual(len(collection.test_results), len(mock_report.mock_calls))
  277. mock_sink_close.assert_called()
  278. @mock.patch('shard_util.shard_index', return_value=0)
  279. @mock.patch('time.time', return_value=10000)
  280. def test_standard_json_output(self, *args):
  281. """Tests standard_json_output."""
  282. passed_test_value = {
  283. 'expected': 'PASS',
  284. 'actual': 'PASS',
  285. 'shard': 0,
  286. 'is_unexpected': False
  287. }
  288. failed_test_value = {
  289. 'expected': 'PASS',
  290. 'actual': 'FAIL FAIL',
  291. 'shard': 0,
  292. 'is_unexpected': True
  293. }
  294. disabled_test_value = {
  295. 'expected': 'SKIP',
  296. 'actual': 'SKIP',
  297. 'shard': 0,
  298. 'is_unexpected': False
  299. }
  300. unexpected_skip_test_value = {
  301. 'expected': 'PASS',
  302. 'actual': 'SKIP',
  303. 'shard': 0,
  304. 'is_unexpected': True
  305. }
  306. crashed_test_value = {
  307. 'expected': 'PASS',
  308. 'actual': 'CRASH',
  309. 'shard': 0,
  310. 'is_unexpected': True
  311. }
  312. flaky_test_value = {
  313. 'expected': 'PASS',
  314. 'actual': 'PASS FAIL',
  315. 'shard': 0,
  316. 'is_unexpected': False,
  317. 'is_flaky': True
  318. }
  319. aborted_test_value = {
  320. 'expected': 'PASS',
  321. 'actual': 'TIMEOUT',
  322. 'shard': 0,
  323. 'is_unexpected': True
  324. }
  325. expected_tests = collections.OrderedDict()
  326. expected_tests['passed/test'] = passed_test_value
  327. expected_tests['failed/test'] = failed_test_value
  328. expected_tests['disabled/test'] = disabled_test_value
  329. expected_tests['unexpected/skipped_test'] = unexpected_skip_test_value
  330. expected_tests['crashed/test'] = crashed_test_value
  331. expected_tests['flaky/test'] = flaky_test_value
  332. expected_tests['aborted/test'] = aborted_test_value
  333. expected_num_failures_by_type = {
  334. 'PASS': 2,
  335. 'FAIL': 1,
  336. 'CRASH': 1,
  337. 'SKIP': 2,
  338. 'TIMEOUT': 1
  339. }
  340. expected_json = {
  341. 'version': 3,
  342. 'path_delimiter': '/',
  343. 'seconds_since_epoch': 10000,
  344. 'interrupted': False,
  345. 'num_failures_by_type': expected_num_failures_by_type,
  346. 'tests': expected_tests
  347. }
  348. self.assertEqual(
  349. self.full_collection.standard_json_output(path_delimiter='/'),
  350. expected_json)
  351. def test_test_runner_logs(self):
  352. """Test test_runner_logs."""
  353. expected_logs = collections.OrderedDict()
  354. expected_logs['passed tests'] = ['passed/test']
  355. expected_logs['disabled tests'] = ['disabled/test']
  356. flaky_logs = ['Failure log of attempt 1:', 'line1', 'line2']
  357. failed_logs = [
  358. 'Failure log of attempt 1:', 'line1', 'line2',
  359. 'Failure log of attempt 2:', 'line3', 'line4'
  360. ]
  361. no_logs = ['Failure log of attempt 1:', '']
  362. expected_logs['flaked tests'] = {'flaky/test': flaky_logs}
  363. expected_logs['failed tests'] = {
  364. 'failed/test': failed_logs,
  365. 'crashed/test': no_logs,
  366. 'unexpected/skipped_test': no_logs,
  367. 'aborted/test': no_logs
  368. }
  369. expected_logs['failed/test'] = failed_logs
  370. expected_logs['unexpected/skipped_test'] = no_logs
  371. expected_logs['flaky/test'] = flaky_logs
  372. expected_logs['crashed/test'] = no_logs
  373. expected_logs['aborted/test'] = no_logs
  374. generated_logs = self.full_collection.test_runner_logs()
  375. keys = [
  376. 'passed tests', 'disabled tests', 'flaked tests', 'failed tests',
  377. 'failed/test', 'unexpected/skipped_test', 'flaky/test', 'crashed/test',
  378. 'aborted/test'
  379. ]
  380. for key in keys:
  381. self.assertEqual(generated_logs[key], expected_logs[key])
  382. if __name__ == '__main__':
  383. unittest.main()