xcode_log_parser_test.py 27 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705
  1. #!/usr/bin/env vpython3
  2. # Copyright 2019 The Chromium Authors. All rights reserved.
  3. # Use of this source code is governed by a BSD-style license that can be
  4. # found in the LICENSE file.
  5. """Unittests for xcode_log_parser.py."""
  6. import json
  7. import mock
  8. import os
  9. import unittest
  10. from test_result_util import TestStatus
  11. import test_runner
  12. import test_runner_test
  13. import xcode_log_parser
  14. OUTPUT_PATH = '/tmp/attempt_0'
  15. XCRESULT_PATH = '/tmp/attempt_0.xcresult'
  16. XCODE11_DICT = {
  17. 'path': '/Users/user1/Xcode.app',
  18. 'version': '11.0',
  19. 'build': '11M336w',
  20. }
  21. # A sample of json result when executing xcresulttool on .xcresult dir without
  22. # --id. Some unused keys and values were removed.
  23. XCRESULT_ROOT = """
  24. {
  25. "_type" : {
  26. "_name" : "ActionsInvocationRecord"
  27. },
  28. "actions" : {
  29. "_values" : [
  30. {
  31. "actionResult" : {
  32. "_type" : {
  33. "_name" : "ActionResult"
  34. },
  35. "diagnosticsRef" : {
  36. "id" : {
  37. "_value" : "DIAGNOSTICS_REF_ID"
  38. }
  39. },
  40. "logRef" : {
  41. "id" : {
  42. "_value" : "0~6jr1GkZxoWVzWfcUNA5feff3l7g8fPHJ1rqKetCBa3QXhCGY74PnEuRwzktleMTFounMfCdDpSr1hRfhUGIUEQ=="
  43. }
  44. },
  45. "testsRef" : {
  46. "id" : {
  47. "_value" : "0~iRbOkDnmtKVIvHSV2jkeuNcg4RDTUaCLZV7KijyxdCqvhqtp08MKxl0MwjBAPpjmruoI7qNHzBR1RJQAlANNHA=="
  48. }
  49. }
  50. }
  51. }
  52. ]
  53. },
  54. "issues" : {
  55. "testFailureSummaries" : {
  56. "_values" : [
  57. {
  58. "documentLocationInCreatingWorkspace" : {
  59. "url" : {
  60. "_value" : "file:\/\/\/..\/..\/ios\/web\/shell\/test\/page_state_egtest.mm#CharacterRangeLen=0&EndingLineNumber=130&StartingLineNumber=130"
  61. }
  62. },
  63. "message" : {
  64. "_value": "Fail. Screenshots: {\\n\\"Failure\\": \\"path.png\\"\\n}"
  65. },
  66. "testCaseName" : {
  67. "_value": "-[PageStateTestCase testZeroContentOffsetAfterLoad]"
  68. }
  69. }
  70. ]
  71. }
  72. },
  73. "metrics" : {
  74. "testsCount" : {
  75. "_value" : "2"
  76. },
  77. "testsFailedCount" : {
  78. "_value" : "1"
  79. }
  80. }
  81. }"""
  82. REF_ID = b"""
  83. {
  84. "actions": {
  85. "_values": [{
  86. "actionResult": {
  87. "testsRef": {
  88. "id": {
  89. "_value": "REF_ID"
  90. }
  91. }
  92. }
  93. }]
  94. }
  95. }"""
  96. # A sample of json result when executing xcresulttool on .xcresult dir with
  97. # "testsRef" as --id input. Some unused keys and values were removed.
  98. TESTS_REF = """
  99. {
  100. "summaries": {
  101. "_values": [{
  102. "testableSummaries": {
  103. "_type": {
  104. "_name": "Array"
  105. },
  106. "_values": [{
  107. "tests": {
  108. "_type": {
  109. "_name": "Array"
  110. },
  111. "_values": [{
  112. "identifier" : {
  113. "_value" : "All tests"
  114. },
  115. "name" : {
  116. "_value" : "All tests"
  117. },
  118. "subtests": {
  119. "_values": [{
  120. "identifier" : {
  121. "_value" : "ios_web_shell_eg2tests_module.xctest"
  122. },
  123. "name" : {
  124. "_value" : "ios_web_shell_eg2tests_module.xctest"
  125. },
  126. "subtests": {
  127. "_values": [{
  128. "identifier" : {
  129. "_value" : "PageStateTestCase"
  130. },
  131. "name" : {
  132. "_value" : "PageStateTestCase"
  133. },
  134. "subtests": {
  135. "_values": [{
  136. "testStatus": {
  137. "_value": "Success"
  138. },
  139. "duration" : {
  140. "_type" : {
  141. "_name" : "Double"
  142. },
  143. "_value" : "35.38412606716156"
  144. },
  145. "identifier": {
  146. "_value": "PageStateTestCase/testMethod1"
  147. },
  148. "name": {
  149. "_value": "testMethod1"
  150. }
  151. },
  152. {
  153. "summaryRef": {
  154. "id": {
  155. "_value": "0~7Q_uAuUSJtx9gtHM08psXFm3g_xiTTg5bpdoDO88nMXo_iMwQTXpqlrlMe5AtkYmnZ7Ux5uEgAe83kJBfoIckw=="
  156. }
  157. },
  158. "testStatus": {
  159. "_value": "Failure"
  160. },
  161. "identifier": {
  162. "_value": "PageStateTestCase\/testZeroContentOffsetAfterLoad"
  163. },
  164. "name": {
  165. "_value": "testZeroContentOffsetAfterLoad"
  166. }
  167. },
  168. {
  169. "testStatus": {
  170. "_value": "Success"
  171. },
  172. "duration" : {
  173. "_type" : {
  174. "_name" : "Double"
  175. },
  176. "_value" : "28.988606716156"
  177. },
  178. "identifier": {
  179. "_value": "PageStateTestCase/testMethod2"
  180. },
  181. "name": {
  182. "_value": "testMethod2"
  183. }
  184. }]
  185. }
  186. }]
  187. }
  188. }]
  189. }
  190. }]
  191. }
  192. }]
  193. }
  194. }]
  195. }
  196. }
  197. """
  198. # A sample of json result when executing xcresulttool on .xcresult dir with
  199. # a single test summaryRef id value as --id input. Some unused keys and values
  200. # were removed.
  201. SINGLE_TEST_SUMMARY_REF = """
  202. {
  203. "_type" : {
  204. "_name" : "ActionTestSummary",
  205. "_supertype" : {
  206. "_name" : "ActionTestSummaryIdentifiableObject",
  207. "_supertype" : {
  208. "_name" : "ActionAbstractTestSummary"
  209. }
  210. }
  211. },
  212. "activitySummaries" : {
  213. "_values" : [
  214. {
  215. "attachments" : {
  216. "_values" : [
  217. {
  218. "filename" : {
  219. "_value" : "Screenshot_25659115-F3E4-47AE-AA34-551C94333D7E.jpg"
  220. },
  221. "payloadRef" : {
  222. "id" : {
  223. "_value" : "SCREENSHOT_REF_ID_1"
  224. }
  225. }
  226. }
  227. ]
  228. },
  229. "title" : {
  230. "_value" : "Start Test at 2020-10-19 14:12:58.111"
  231. }
  232. },
  233. {
  234. "subactivities" : {
  235. "_values" : [
  236. {
  237. "attachments" : {
  238. "_values" : [
  239. {
  240. "filename" : {
  241. "_value" : "Screenshot_23D95D0E-8B97-4F99-BE3C-A46EDE5999D7.jpg"
  242. },
  243. "payloadRef" : {
  244. "id" : {
  245. "_value" : "SCREENSHOT_REF_ID_2"
  246. }
  247. }
  248. }
  249. ]
  250. },
  251. "subactivities" : {
  252. "_values" : [
  253. {
  254. "subactivities" : {
  255. "_values" : [
  256. {
  257. "attachments" : {
  258. "_values" : [
  259. {
  260. "filename" : {
  261. "_value" : "Crash_3F0A2B1C-7ADA-436E-A54C-D4C39B8411F8.crash"
  262. },
  263. "payloadRef" : {
  264. "id" : {
  265. "_value" : "CRASH_REF_ID_IN_ACTIVITY_SUMMARIES"
  266. }
  267. }
  268. }
  269. ]
  270. },
  271. "title" : {
  272. "_value" : "Wait for org.chromium.ios-web-shell-eg2tests to idle"
  273. }
  274. }
  275. ]
  276. },
  277. "title" : {
  278. "_value" : "Activate org.chromium.ios-web-shell-eg2tests"
  279. }
  280. }
  281. ]
  282. },
  283. "title" : {
  284. "_value" : "Open org.chromium.ios-web-shell-eg2tests"
  285. }
  286. }
  287. ]
  288. },
  289. "title" : {
  290. "_value" : "Set Up"
  291. }
  292. },
  293. {
  294. "title" : {
  295. "_value" : "Find the Target Application 'org.chromium.ios-web-shell-eg2tests'"
  296. }
  297. },
  298. {
  299. "attachments" : {
  300. "_values" : [
  301. {
  302. "filename" : {
  303. "_value" : "Screenshot_278BA84B-2196-4CCD-9D31-2C07DDDC9DFC.jpg"
  304. },
  305. "payloadRef" : {
  306. "id" : {
  307. "_value" : "SCREENSHOT_REF_ID_3"
  308. }
  309. }
  310. }
  311. ]
  312. },
  313. "title" : {
  314. "_value" : "Uncaught Exception at page_state_egtest.mm:131: \\nCannot scroll, the..."
  315. }
  316. },
  317. {
  318. "title" : {
  319. "_value" : "Uncaught Exception: Immediately halt execution of testcase (EarlGreyInternalTestInterruptException)"
  320. }
  321. },
  322. {
  323. "title" : {
  324. "_value" : "Tear Down"
  325. }
  326. }
  327. ]
  328. },
  329. "failureSummaries" : {
  330. "_values" : [
  331. {
  332. "attachments" : {
  333. "_values" : [
  334. {
  335. "filename" : {
  336. "_value" : "kXCTAttachmentLegacyScreenImageData_1_6CED1FE5-96CA-47EA-9852-6FADED687262.jpeg"
  337. },
  338. "payloadRef" : {
  339. "id" : {
  340. "_value" : "SCREENSHOT_REF_ID_IN_FAILURE_SUMMARIES"
  341. }
  342. }
  343. }
  344. ]
  345. },
  346. "fileName" : {
  347. "_value" : "\/..\/..\/ios\/web\/shell\/test\/page_state_egtest.mm"
  348. },
  349. "lineNumber" : {
  350. "_value" : "131"
  351. },
  352. "message" : {
  353. "_value" : "Some logs."
  354. }
  355. },
  356. {
  357. "message" : {
  358. "_value" : "Immediately halt execution of testcase (EarlGreyInternalTestInterruptException)"
  359. }
  360. }
  361. ]
  362. },
  363. "identifier" : {
  364. "_value" : "PageStateTestCase\/testZeroContentOffsetAfterLoad"
  365. },
  366. "name" : {
  367. "_value" : "testZeroContentOffsetAfterLoad"
  368. },
  369. "testStatus" : {
  370. "_value" : "Failure"
  371. }
  372. }"""
  373. def _xcresulttool_get_side_effect(xcresult_path, ref_id=None):
  374. """Side effect for _xcresulttool_get in Xcode11LogParser tested."""
  375. if ref_id is None:
  376. return XCRESULT_ROOT
  377. if ref_id == 'testsRef':
  378. return TESTS_REF
  379. # Other situation in use cases of xcode_log_parser is asking for single test
  380. # summary ref.
  381. return SINGLE_TEST_SUMMARY_REF
  382. class UtilMethodsTest(test_runner_test.TestCase):
  383. """Test case for utility methods not related with Parser class."""
  384. def testParseTestsForInterruptedRun(self):
  385. test_output = """
  386. Test case '-[DownloadManagerTestCase testVisibleFileNameAndOpenInDownloads]' passed on 'Clone 2 of iPhone X 15.0 test simulator - ios_chrome_ui_eg2tests_module-Runner (34498)' (20.715 seconds)
  387. Test case '-[SyncFakeServerTestCase testSyncDownloadBookmark]' passed on 'Clone 1 of iPhone X 15.0 test simulator - ios_chrome_ui_eg2tests_module-Runner (34249)' (14.880 seconds)
  388. Random lines
  389. t = 53.90s Tear Down
  390. Test Case '-[LinkToTextTestCase testGenerateLinkForSimpleText]' failed (55.316 seconds).
  391. t = nans Suite Tear Down
  392. Test Suite 'LinkToTextTestCase' failed at 2021-06-15 07:13:17.406.
  393. Executed 1 test, with 6 failures (6 unexpected) in 55.316 (55.338) seconds
  394. Test Suite 'ios_chrome_ui_eg2tests_module.xctest' failed at 2021-06-15 07:13:17.407.
  395. Executed 1 test, with 6 failures (6 unexpected) in 55.316 (55.340) seconds
  396. Test Suite 'Selected tests' failed at 2021-06-15 07:13:17.408.
  397. Executed 1 test, with 6 failures (6 unexpected) in 55.316 (55.342) seconds
  398. """
  399. test_output_list = test_output.split('\n')
  400. expected_passed = set([
  401. 'DownloadManagerTestCase/testVisibleFileNameAndOpenInDownloads',
  402. 'SyncFakeServerTestCase/testSyncDownloadBookmark'
  403. ])
  404. expected_failed = set(['LinkToTextTestCase/testGenerateLinkForSimpleText'])
  405. expected_failed_message = 'Test failed in interrupted(timedout) run.'
  406. results = xcode_log_parser.parse_passed_failed_tests_for_interrupted_run(
  407. test_output_list)
  408. self.assertEqual(results.expected_tests(), expected_passed)
  409. self.assertEqual(results.unexpected_tests(), expected_failed)
  410. for result in results.test_results:
  411. if result.name == 'LinkToTextTestCase/testGenerateLinkForSimpleText':
  412. self.assertEqual(result.test_log, expected_failed_message)
  413. class XCode11LogParserTest(test_runner_test.TestCase):
  414. """Test case to test Xcode11LogParser."""
  415. def setUp(self):
  416. super(XCode11LogParserTest, self).setUp()
  417. self.mock(test_runner, 'get_current_xcode_info', lambda: XCODE11_DICT)
  418. @mock.patch('xcode_util.version', autospec=True)
  419. def testGetParser(self, mock_xcode_version):
  420. mock_xcode_version.return_value = ('12.0', '12A7209')
  421. self.assertEqual(xcode_log_parser.get_parser().__class__.__name__,
  422. 'Xcode11LogParser')
  423. mock_xcode_version.return_value = ('11.4', '11E146')
  424. self.assertEqual(xcode_log_parser.get_parser().__class__.__name__,
  425. 'Xcode11LogParser')
  426. mock_xcode_version.return_value = ('10.3', '10G8')
  427. self.assertEqual(xcode_log_parser.get_parser().__class__.__name__,
  428. 'XcodeLogParser')
  429. @mock.patch('subprocess.check_output', autospec=True)
  430. def testXcresulttoolGetRoot(self, mock_process):
  431. mock_process.return_value = b'%JSON%'
  432. xcode_log_parser.Xcode11LogParser()._xcresulttool_get('xcresult_path')
  433. self.assertTrue(
  434. os.path.join(XCODE11_DICT['path'], 'usr', 'bin') in os.environ['PATH'])
  435. self.assertEqual(
  436. ['xcresulttool', 'get', '--format', 'json', '--path', 'xcresult_path'],
  437. mock_process.mock_calls[0][1][0])
  438. @mock.patch('subprocess.check_output', autospec=True)
  439. def testXcresulttoolGetRef(self, mock_process):
  440. mock_process.side_effect = [REF_ID, b'JSON']
  441. xcode_log_parser.Xcode11LogParser()._xcresulttool_get('xcresult_path',
  442. 'testsRef')
  443. self.assertEqual(
  444. ['xcresulttool', 'get', '--format', 'json', '--path', 'xcresult_path'],
  445. mock_process.mock_calls[0][1][0])
  446. self.assertEqual([
  447. 'xcresulttool', 'get', '--format', 'json', '--path', 'xcresult_path',
  448. '--id', 'REF_ID'], mock_process.mock_calls[1][1][0])
  449. def testXcresulttoolListFailedTests(self):
  450. failure_message = (
  451. 'file:///../../ios/web/shell/test/page_state_egtest.mm#'
  452. 'CharacterRangeLen=0&EndingLineNumber=130&StartingLineNumber=130\n'
  453. 'Fail. Screenshots: {\n\"Failure\": \"path.png\"\n}')
  454. expected = set(['PageStateTestCase/testZeroContentOffsetAfterLoad'])
  455. results = xcode_log_parser.Xcode11LogParser()._list_of_failed_tests(
  456. json.loads(XCRESULT_ROOT))
  457. self.assertEqual(expected, results.failed_tests())
  458. log = results.test_results[0].test_log
  459. self.assertEqual(log, failure_message)
  460. def testXcresulttoolListFailedTestsExclude(self):
  461. excluded = set(['PageStateTestCase/testZeroContentOffsetAfterLoad'])
  462. results = xcode_log_parser.Xcode11LogParser()._list_of_failed_tests(
  463. json.loads(XCRESULT_ROOT), excluded=excluded)
  464. self.assertEqual(set([]), results.all_test_names())
  465. @mock.patch('xcode_log_parser.Xcode11LogParser._export_data')
  466. @mock.patch('xcode_log_parser.Xcode11LogParser._xcresulttool_get')
  467. def testGetTestStatuses(self, mock_xcresult, mock_export):
  468. mock_xcresult.side_effect = _xcresulttool_get_side_effect
  469. # self.assertEqual(test_result.test_log, lo
  470. expected_failure_log = (
  471. 'Logs from "failureSummaries" in .xcresult:\n'
  472. 'file: /../../ios/web/shell/test/page_state_egtest.mm, line: 131\n'
  473. 'Some logs.\n'
  474. 'file: , line: \n'
  475. 'Immediately halt execution of testcase '
  476. '(EarlGreyInternalTestInterruptException)\n')
  477. expected_expected_tests = set(
  478. ['PageStateTestCase/testMethod1', 'PageStateTestCase/testMethod2'])
  479. results = xcode_log_parser.Xcode11LogParser()._get_test_statuses(
  480. OUTPUT_PATH)
  481. self.assertEqual(expected_expected_tests, results.expected_tests())
  482. seen_failed_test = False
  483. for test_result in results.test_results:
  484. if test_result.name == 'PageStateTestCase/testZeroContentOffsetAfterLoad':
  485. seen_failed_test = True
  486. self.assertEqual(test_result.test_log, expected_failure_log)
  487. self.assertEqual(test_result.duration, None)
  488. crash_file_name = (
  489. 'attempt_0_PageStateTestCase_testZeroContentOffsetAfterLoad_'
  490. 'Crash_3F0A2B1C-7ADA-436E-A54C-D4C39B8411F8.crash'
  491. )
  492. jpeg_file_name = (
  493. 'attempt_0_PageStateTestCase_testZeroContentOffsetAfterLoad'
  494. '_kXCTAttachmentLegacyScreenImageData_1'
  495. '_6CED1FE5-96CA-47EA-9852-6FADED687262.jpeg')
  496. self.assertDictEqual(
  497. {
  498. crash_file_name: '/tmp/%s' % crash_file_name,
  499. jpeg_file_name: '/tmp/%s' % jpeg_file_name,
  500. }, test_result.attachments)
  501. if test_result.name == 'PageStateTestCase/testMethod1':
  502. self.assertEqual(test_result.duration, 35384)
  503. if test_result.name == 'PageStateTestCase/testMethod2':
  504. self.assertEqual(test_result.duration, 28988)
  505. self.assertTrue(seen_failed_test)
  506. @mock.patch('file_util.zip_and_remove_folder')
  507. @mock.patch('xcode_log_parser.Xcode11LogParser._extract_artifacts_for_test')
  508. @mock.patch('xcode_log_parser.Xcode11LogParser.export_diagnostic_data')
  509. @mock.patch('os.path.exists', autospec=True)
  510. @mock.patch('xcode_log_parser.Xcode11LogParser._xcresulttool_get')
  511. def testCollectTestTesults(self, mock_root, mock_exist_file, *args):
  512. expected_passed = set(
  513. ['PageStateTestCase/testMethod1', 'PageStateTestCase/testMethod2'])
  514. expected_failed = set(['PageStateTestCase/testZeroContentOffsetAfterLoad'])
  515. mock_root.side_effect = _xcresulttool_get_side_effect
  516. mock_exist_file.return_value = True
  517. results = xcode_log_parser.Xcode11LogParser().collect_test_results(
  518. OUTPUT_PATH, [])
  519. # Length ensures no duplicate results from |_get_test_statuses| and
  520. # |_list_of_failed_tests|.
  521. self.assertEqual(len(results.test_results), 3)
  522. self.assertEqual(expected_passed, results.expected_tests())
  523. self.assertEqual(expected_failed, results.unexpected_tests())
  524. # Ensure format.
  525. for test in results.test_results:
  526. self.assertTrue(isinstance(test.name, str))
  527. if test.status == TestStatus.FAIL:
  528. self.assertTrue(isinstance(test.test_log, str))
  529. @mock.patch('file_util.zip_and_remove_folder')
  530. @mock.patch('xcode_log_parser.Xcode11LogParser.copy_artifacts')
  531. @mock.patch('xcode_log_parser.Xcode11LogParser.export_diagnostic_data')
  532. @mock.patch('os.path.exists', autospec=True)
  533. @mock.patch('xcode_log_parser.Xcode11LogParser._xcresulttool_get')
  534. def testCollectTestsRanZeroTests(self, mock_root, mock_exist_file, *args):
  535. metrics_json = '{"metrics": {}}'
  536. mock_root.return_value = metrics_json
  537. mock_exist_file.return_value = True
  538. results = xcode_log_parser.Xcode11LogParser().collect_test_results(
  539. OUTPUT_PATH, [])
  540. self.assertTrue(results.crashed)
  541. self.assertEqual(results.crash_message, '0 tests executed!')
  542. self.assertEqual(len(results.all_test_names()), 0)
  543. @mock.patch('os.path.exists', autospec=True)
  544. def testCollectTestsDidNotRun(self, mock_exist_file):
  545. mock_exist_file.return_value = False
  546. results = xcode_log_parser.Xcode11LogParser().collect_test_results(
  547. OUTPUT_PATH, [])
  548. self.assertTrue(results.crashed)
  549. self.assertEqual(results.crash_message,
  550. '/tmp/attempt_0 with staging data does not exist.\n')
  551. self.assertEqual(len(results.all_test_names()), 0)
  552. @mock.patch('os.path.exists', autospec=True)
  553. def testCollectTestsInterruptedRun(self, mock_exist_file):
  554. mock_exist_file.side_effect = [True, False]
  555. results = xcode_log_parser.Xcode11LogParser().collect_test_results(
  556. OUTPUT_PATH, [])
  557. self.assertTrue(results.crashed)
  558. self.assertEqual(
  559. results.crash_message,
  560. '/tmp/attempt_0.xcresult with test results does not exist.\n')
  561. self.assertEqual(len(results.all_test_names()), 0)
  562. @mock.patch('subprocess.check_output', autospec=True)
  563. @mock.patch('os.path.exists', autospec=True)
  564. @mock.patch('xcode_log_parser.Xcode11LogParser._xcresulttool_get')
  565. def testCopyScreenshots(self, mock_xcresulttool_get, mock_path_exists,
  566. mock_process):
  567. mock_path_exists.return_value = True
  568. mock_xcresulttool_get.side_effect = _xcresulttool_get_side_effect
  569. xcode_log_parser.Xcode11LogParser().copy_artifacts(OUTPUT_PATH)
  570. mock_process.assert_any_call([
  571. 'xcresulttool', 'export', '--type', 'file', '--id',
  572. 'SCREENSHOT_REF_ID_IN_FAILURE_SUMMARIES', '--path', XCRESULT_PATH,
  573. '--output-path',
  574. '/tmp/attempt_0_PageStateTestCase_testZeroContentOffsetAfterLoad'
  575. '_kXCTAttachmentLegacyScreenImageData_1'
  576. '_6CED1FE5-96CA-47EA-9852-6FADED687262.jpeg'
  577. ])
  578. mock_process.assert_any_call([
  579. 'xcresulttool', 'export', '--type', 'file', '--id',
  580. 'CRASH_REF_ID_IN_ACTIVITY_SUMMARIES', '--path', XCRESULT_PATH,
  581. '--output-path',
  582. '/tmp/attempt_0_PageStateTestCase_testZeroContentOffsetAfterLoad'
  583. '_Crash_3F0A2B1C-7ADA-436E-A54C-D4C39B8411F8.crash'
  584. ])
  585. # Ensures screenshots in activitySummaries are not copied.
  586. self.assertEqual(2, mock_process.call_count)
  587. @mock.patch('file_util.zip_and_remove_folder')
  588. @mock.patch('subprocess.check_output', autospec=True)
  589. @mock.patch('os.path.exists', autospec=True)
  590. @mock.patch('xcode_log_parser.Xcode11LogParser._xcresulttool_get')
  591. def testExportDiagnosticData(self, mock_xcresulttool_get, mock_path_exists,
  592. mock_process, _):
  593. mock_path_exists.return_value = True
  594. mock_xcresulttool_get.side_effect = _xcresulttool_get_side_effect
  595. xcode_log_parser.Xcode11LogParser.export_diagnostic_data(OUTPUT_PATH)
  596. mock_process.assert_called_with([
  597. 'xcresulttool', 'export', '--type', 'directory', '--id',
  598. 'DIAGNOSTICS_REF_ID', '--path', XCRESULT_PATH, '--output-path',
  599. '/tmp/attempt_0.xcresult_diagnostic'
  600. ])
  601. @mock.patch('file_util.zip_and_remove_folder')
  602. @mock.patch('shutil.copy')
  603. @mock.patch('subprocess.check_output', autospec=True)
  604. @mock.patch('os.path.exists', autospec=True)
  605. @mock.patch('xcode_log_parser.Xcode11LogParser._xcresulttool_get')
  606. def testStdoutCopiedInExportDiagnosticData(self, mock_xcresulttool_get,
  607. mock_path_exists, mock_process,
  608. mock_copy, _):
  609. output_path_in_test = 'test_data/attempt_0'
  610. xcresult_path_in_test = 'test_data/attempt_0.xcresult'
  611. mock_path_exists.return_value = True
  612. mock_xcresulttool_get.side_effect = _xcresulttool_get_side_effect
  613. xcode_log_parser.Xcode11LogParser.export_diagnostic_data(
  614. output_path_in_test)
  615. # os.walk() walks folders in unknown sequence. Use try-except blocks to
  616. # assert that any of the 2 assertions is true.
  617. try:
  618. mock_copy.assert_any_call(
  619. 'test_data/attempt_0.xcresult_diagnostic/test_module-UUID/test_module-UUID1/StandardOutputAndStandardError.txt',
  620. 'test_data/attempt_0/../attempt_0_simulator#1_StandardOutputAndStandardError.txt'
  621. )
  622. except AssertionError:
  623. mock_copy.assert_any_call(
  624. 'test_data/attempt_0.xcresult_diagnostic/test_module-UUID/test_module-UUID1/StandardOutputAndStandardError.txt',
  625. 'test_data/attempt_0/../attempt_0_simulator#0_StandardOutputAndStandardError.txt'
  626. )
  627. try:
  628. mock_copy.assert_any_call(
  629. 'test_data/attempt_0.xcresult_diagnostic/test_module-UUID/test_module-UUID2/StandardOutputAndStandardError-org.chromium.gtest.ios-chrome-eg2tests.txt',
  630. 'test_data/attempt_0/../attempt_0_simulator#1_StandardOutputAndStandardError-org.chromium.gtest.ios-chrome-eg2tests.txt'
  631. )
  632. except AssertionError:
  633. mock_copy.assert_any_call(
  634. 'test_data/attempt_0.xcresult_diagnostic/test_module-UUID/test_module-UUID2/StandardOutputAndStandardError-org.chromium.gtest.ios-chrome-eg2tests.txt',
  635. 'test_data/attempt_0/../attempt_0_simulator#0_StandardOutputAndStandardError-org.chromium.gtest.ios-chrome-eg2tests.txt'
  636. )
  637. @mock.patch('os.path.exists', autospec=True)
  638. def testCollectTestResults_interruptedTests(self, mock_path_exists):
  639. mock_path_exists.side_effect = [True, False]
  640. output = [
  641. '[09:03:42:INFO] Test case \'-[TestCase1 method1]\' passed on device.',
  642. '[09:06:40:INFO] Test Case \'-[TestCase2 method1]\' passed on device.',
  643. '[09:09:00:INFO] Test case \'-[TestCase2 method1]\' failed on device.',
  644. '** BUILD INTERRUPTED **',
  645. ]
  646. not_found_message = ['%s with test results does not exist.' % XCRESULT_PATH]
  647. res = xcode_log_parser.Xcode11LogParser().collect_test_results(
  648. OUTPUT_PATH, output)
  649. self.assertTrue(res.crashed)
  650. self.assertEqual('\n'.join(not_found_message + output), res.crash_message)
  651. self.assertEqual(
  652. set(['TestCase1/method1', 'TestCase2/method1']), res.expected_tests())
  653. @mock.patch('file_util.zip_and_remove_folder')
  654. @mock.patch('xcode_log_parser.Xcode11LogParser._extract_artifacts_for_test')
  655. @mock.patch('xcode_log_parser.Xcode11LogParser.export_diagnostic_data')
  656. @mock.patch('os.path.exists', autospec=True)
  657. @mock.patch('xcode_log_parser.Xcode11LogParser._xcresulttool_get')
  658. @mock.patch('xcode_log_parser.Xcode11LogParser._list_of_failed_tests')
  659. def testArtifactsDiagnosticLogsExportedInCollectTestTesults(
  660. self, mock_get_failed_tests, mock_root, mock_exist_file,
  661. mock_export_diagnostic_data, mock_extract_artifacts, mock_zip):
  662. mock_root.side_effect = _xcresulttool_get_side_effect
  663. mock_exist_file.return_value = True
  664. xcode_log_parser.Xcode11LogParser().collect_test_results(OUTPUT_PATH, [])
  665. mock_export_diagnostic_data.assert_called_with(OUTPUT_PATH)
  666. mock_extract_artifacts.assert_called()
  667. if __name__ == '__main__':
  668. unittest.main()