gtest_utils_test.py 36 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969
  1. #!/usr/bin/env vpython3
  2. # Copyright 2021 The Chromium Authors. All rights reserved.
  3. # Use of this source code is governed by a BSD-style license that can be
  4. # found in the LICENSE file.
  5. """Unit tests for classes in gtest_utils.py."""
  6. import unittest
  7. import gtest_utils
  8. from test_result_util import TestStatus
  9. FAILURES = [
  10. 'NavigationControllerTest.Reload',
  11. 'NavigationControllerTest/SpdyNetworkTransTest.Constructor/0',
  12. 'BadTest.TimesOut', 'MoreBadTest.TimesOutAndFails',
  13. 'SomeOtherTest.SwitchTypes', 'SomeOtherTest.FAILS_ThisTestTimesOut'
  14. ]
  15. FAILS_FAILURES = ['SomeOtherTest.FAILS_Bar']
  16. FLAKY_FAILURES = ['SomeOtherTest.FLAKY_Baz']
  17. CRASH_MESSAGE = ['Oops, this test crashed!']
  18. TIMEOUT_MESSAGE = 'Killed (timed out).'
  19. RELOAD_ERRORS = (r'C:\b\slave\chrome-release-snappy\build\chrome\browser'
  20. r'\navigation_controller_unittest.cc:381: Failure' + """
  21. Value of: -1
  22. Expected: contents->controller()->GetPendingEntryIndex()
  23. Which is: 0
  24. """)
  25. SPDY_ERRORS = (r'C:\b\slave\chrome-release-snappy\build\chrome\browser'
  26. r'\navigation_controller_unittest.cc:439: Failure' + """
  27. Value of: -1
  28. Expected: contents->controller()->GetPendingEntryIndex()
  29. Which is: 0
  30. """)
  31. SWITCH_ERRORS = (r'C:\b\slave\chrome-release-snappy\build\chrome\browser'
  32. r'\navigation_controller_unittest.cc:615: Failure' + """
  33. Value of: -1
  34. Expected: contents->controller()->GetPendingEntryIndex()
  35. Which is: 0
  36. """ + r'C:\b\slave\chrome-release-snappy\build\chrome\browser'
  37. r'\navigation_controller_unittest.cc:617: Failure' + """
  38. Value of: contents->controller()->GetPendingEntry()
  39. Actual: true
  40. Expected: false
  41. """)
  42. # pylint: disable=line-too-long
  43. TIMEOUT_ERRORS = (
  44. '[61613:263:0531/042613:2887943745568888:ERROR:/b/slave'
  45. '/chromium-rel-mac-builder/build/src/chrome/browser/extensions'
  46. '/extension_error_reporter.cc(56)] Extension error: Could not load extension '
  47. 'from \'extensions/api_test/geolocation/no_permission\'. Manifest file is '
  48. 'missing or unreadable.')
  49. MOREBAD_ERRORS = """
  50. Value of: entry->page_type()
  51. Actual: 2
  52. Expected: NavigationEntry::NORMAL_PAGE
  53. """
  54. TEST_DATA = (
  55. """
  56. [==========] Running 7 tests from 3 test cases.
  57. [----------] Global test environment set-up.
  58. [----------] 1 test from HunspellTest
  59. [ RUN ] HunspellTest.All
  60. [ OK ] HunspellTest.All (62 ms)
  61. [----------] 1 test from HunspellTest (62 ms total)
  62. [----------] 4 tests from NavigationControllerTest
  63. [ RUN ] NavigationControllerTest.Defaults
  64. [ OK ] NavigationControllerTest.Defaults (48 ms)
  65. [ RUN ] NavigationControllerTest.Reload
  66. %(reload_errors)s
  67. [ FAILED ] NavigationControllerTest.Reload (2 ms)
  68. [ RUN ] NavigationControllerTest.Reload_GeneratesNewPage
  69. [ OK ] NavigationControllerTest.Reload_GeneratesNewPage (22 ms)
  70. [ RUN ] NavigationControllerTest/SpdyNetworkTransTest.Constructor/0
  71. %(spdy_errors)s
  72. [ FAILED ] NavigationControllerTest/SpdyNetworkTransTest.Constructor/0 (2 ms)
  73. [----------] 4 tests from NavigationControllerTest (74 ms total)
  74. YOU HAVE 2 FLAKY TESTS
  75. [----------] 1 test from BadTest
  76. [ RUN ] BadTest.TimesOut
  77. %(timeout_errors)s
  78. """ % {
  79. 'reload_errors': RELOAD_ERRORS,
  80. 'spdy_errors': SPDY_ERRORS,
  81. 'timeout_errors': TIMEOUT_ERRORS
  82. } + '[0531/042642:ERROR:/b/slave/chromium-rel-mac-builder/build/src/chrome'
  83. '/test/test_launcher/out_of_proc_test_runner.cc(79)] Test timeout (30000 ms) '
  84. 'exceeded for BadTest.TimesOut' + """
  85. Handling SIGTERM.
  86. Successfully wrote to shutdown pipe, resetting signal handler.
  87. """ +
  88. '[61613:19971:0531/042642:2887973024284693:INFO:/b/slave/chromium-rel-mac-'
  89. 'builder/build/src/chrome/browser/browser_main.cc(285)] Handling shutdown for '
  90. 'signal 15.' + """
  91. [----------] 1 test from MoreBadTest
  92. [ RUN ] MoreBadTest.TimesOutAndFails
  93. %(morebad_errors)s
  94. """ % {
  95. 'morebad_errors': MOREBAD_ERRORS
  96. } +
  97. '[0531/042642:ERROR:/b/slave/chromium-rel-mac-builder/build/src/chrome/test'
  98. '/test_launcher/out_of_proc_test_runner.cc(79)] Test timeout (30000 ms) '
  99. 'exceeded for MoreBadTest.TimesOutAndFails' + """
  100. Handling SIGTERM.
  101. Successfully wrote to shutdown pipe, resetting signal handler.
  102. [ FAILED ] MoreBadTest.TimesOutAndFails (31000 ms)
  103. [----------] 5 tests from SomeOtherTest
  104. [ RUN ] SomeOtherTest.SwitchTypes
  105. %(switch_errors)s
  106. [ FAILED ] SomeOtherTest.SwitchTypes (40 ms)
  107. [ RUN ] SomeOtherTest.Foo
  108. [ OK ] SomeOtherTest.Foo (20 ms)
  109. [ RUN ] SomeOtherTest.FAILS_Bar
  110. Some error message for a failing test.
  111. [ FAILED ] SomeOtherTest.FAILS_Bar (40 ms)
  112. [ RUN ] SomeOtherTest.FAILS_ThisTestTimesOut
  113. """ % {
  114. 'switch_errors': SWITCH_ERRORS
  115. } + '[0521/041343:ERROR:test_launcher.cc(384)] Test timeout (5000 ms) '
  116. 'exceeded for SomeOtherTest.FAILS_ThisTestTimesOut' + """
  117. [ RUN ] SomeOtherTest.FLAKY_Baz
  118. Some error message for a flaky test.
  119. [ FAILED ] SomeOtherTest.FLAKY_Baz (40 ms)
  120. [----------] 2 tests from SomeOtherTest (60 ms total)
  121. [----------] Global test environment tear-down
  122. [==========] 8 tests from 3 test cases ran. (3750 ms total)
  123. [ PASSED ] 4 tests.
  124. [ FAILED ] 4 tests, listed below:
  125. [ FAILED ] NavigationControllerTest.Reload
  126. [ FAILED ] NavigationControllerTest/SpdyNetworkTransTest.Constructor/0
  127. [ FAILED ] SomeOtherTest.SwitchTypes
  128. [ FAILED ] SomeOtherTest.FAILS_ThisTestTimesOut
  129. 1 FAILED TEST
  130. YOU HAVE 10 DISABLED TESTS
  131. YOU HAVE 2 FLAKY TESTS
  132. program finished with exit code 1
  133. """)
  134. TEST_DATA_CRASH = """
  135. [==========] Running 7 tests from 3 test cases.
  136. [----------] Global test environment set-up.
  137. [----------] 1 test from HunspellTest
  138. [ RUN ] HunspellTest.Crashes
  139. Oops, this test crashed!
  140. """
  141. TEST_DATA_MIXED_STDOUT = """
  142. [==========] Running 3 tests from 3 test cases.
  143. [----------] Global test environment set-up.
  144. [----------] 1 tests from WebSocketHandshakeHandlerSpdy3Test
  145. [ RUN ] WebSocketHandshakeHandlerSpdy3Test.RequestResponse
  146. [ OK ] WebSocketHandshakeHandlerSpdy3Test.RequestResponse (1 ms)
  147. [----------] 1 tests from WebSocketHandshakeHandlerSpdy3Test (1 ms total)
  148. [----------] 1 test from URLRequestTestFTP
  149. [ RUN ] URLRequestTestFTP.UnsafePort
  150. FTP server started on port 32841...
  151. sending server_data: {"host": "127.0.0.1", "port": 32841} (36 bytes)
  152. starting FTP server[ OK ] URLRequestTestFTP.UnsafePort (300 ms)
  153. [----------] 1 test from URLRequestTestFTP (300 ms total)
  154. [ RUN ] TestFix.TestCase
  155. [1:2/3:WARNING:extension_apitest.cc(169)] Workaround for 177163,
  156. prematurely stopping test
  157. [ OK ] X (1000ms total)
  158. [----------] 1 test from Crash
  159. [ RUN ] Crash.Test
  160. Oops, this test crashed!
  161. """
  162. TEST_DATA_SKIPPED = """
  163. [==========] Running 1 test from 1 test suite.
  164. [----------] Global test environment set-up.
  165. [----------] 1 test from ProcessReaderLinux
  166. [ RUN ] ProcessReaderLinux.AbortMessage
  167. ../../third_party/crashpad/crashpad/snapshot/linux/process_reader_linux_test.cc:842: Skipped
  168. Stack trace:
  169. #00 pc 0x00000000002350b7 /data/local/tmp/crashpad_tests__dist/crashpad_tests
  170. #01 pc 0x0000000000218183 /data/local/tmp/crashpad_tests__dist/crashpad_tests
  171. [ SKIPPED ] ProcessReaderLinux.AbortMessage (1 ms)
  172. [----------] 1 test from ProcessReaderLinux (2 ms total)
  173. [----------] Global test environment tear-down
  174. [==========] 1 test from 1 test suite ran. (2 ms total)
  175. [ PASSED ] 0 tests.
  176. [ SKIPPED ] 1 test, listed below:
  177. [ SKIPPED ] ProcessReaderLinux.AbortMessage
  178. """
  179. VALGRIND_HASH = 'B254345E4D3B6A00'
  180. VALGRIND_REPORT = """Leak_DefinitelyLost
  181. 1 (1 direct, 0 indirect) bytes in 1 blocks are lost in loss record 1 of 1
  182. operator new(unsigned long) (m_replacemalloc/vg_replace_malloc.c:1140)
  183. content::NavigationControllerTest_Reload::TestBody() (a/b/c/d.cc:1150)
  184. Suppression (error hash=#%(hash)s#):
  185. {
  186. <insert_a_suppression_name_here>
  187. Memcheck:Leak
  188. fun:_Znw*
  189. fun:_ZN31NavigationControllerTest_Reload8TestBodyEv
  190. }""" % {
  191. 'hash': VALGRIND_HASH
  192. }
  193. TEST_DATA_VALGRIND = """
  194. [==========] Running 5 tests from 2 test cases.
  195. [----------] Global test environment set-up.
  196. [----------] 1 test from HunspellTest
  197. [ RUN ] HunspellTest.All
  198. [ OK ] HunspellTest.All (62 ms)
  199. [----------] 1 test from HunspellTest (62 ms total)
  200. [----------] 4 tests from NavigationControllerTest
  201. [ RUN ] NavigationControllerTest.Defaults
  202. [ OK ] NavigationControllerTest.Defaults (48 ms)
  203. [ RUN ] NavigationControllerTest.Reload
  204. [ OK ] NavigationControllerTest.Reload (2 ms)
  205. [ RUN ] NavigationControllerTest.Reload_GeneratesNewPage
  206. [ OK ] NavigationControllerTest.Reload_GeneratesNewPage (22 ms)
  207. [ RUN ] NavigationControllerTest/SpdyNetworkTransTest.Constructor/0
  208. [ OK ] NavigationControllerTest/SpdyNetworkTransTest.Constructor/0 (2 ms)
  209. [----------] 4 tests from NavigationControllerTest (74 ms total)
  210. [----------] Global test environment tear-down
  211. [==========] 5 tests from 1 test cases ran. (136 ms total)
  212. [ PASSED ] 5 tests.
  213. ### BEGIN MEMORY TOOL REPORT (error hash=#%(hash)s#)
  214. %(report)s
  215. ### END MEMORY TOOL REPORT (error hash=#%(hash)s#)
  216. program finished with exit code 255
  217. """ % {
  218. 'report': VALGRIND_REPORT,
  219. 'hash': VALGRIND_HASH
  220. }
  221. FAILING_TESTS_OUTPUT = """
  222. Failing tests:
  223. ChromeRenderViewTest.FAILS_AllowDOMStorage
  224. PrerenderBrowserTest.PrerenderHTML5VideoJs
  225. """
  226. FAILING_TESTS_EXPECTED = [
  227. 'ChromeRenderViewTest.FAILS_AllowDOMStorage',
  228. 'PrerenderBrowserTest.PrerenderHTML5VideoJs'
  229. ]
  230. TEST_DATA_SHARD_0 = (
  231. """Note: This is test shard 1 of 30.
  232. [==========] Running 6 tests from 3 test cases.
  233. [----------] Global test environment set-up.
  234. [----------] 1 test from HunspellTest
  235. [ RUN ] HunspellTest.All
  236. [ OK ] HunspellTest.All (62 ms)
  237. [----------] 1 test from HunspellTest (62 ms total)
  238. [----------] 1 test from BadTest
  239. [ RUN ] BadTest.TimesOut
  240. %(timeout_errors)s
  241. """ % {
  242. 'timeout_errors': TIMEOUT_ERRORS
  243. } +
  244. '[0531/042642:ERROR:/b/slave/chromium-rel-mac-builder/build/src/chrome/test'
  245. '/test_launcher/out_of_proc_test_runner.cc(79)] Test timeout (30000 ms) '
  246. 'exceeded for BadTest.TimesOut' + """
  247. Handling SIGTERM.
  248. Successfully wrote to shutdown pipe, resetting signal handler.
  249. """ +
  250. '[61613:19971:0531/042642:2887973024284693:INFO:/b/slave/chromium-rel-mac-'
  251. 'builder/build/src/chrome/browser/browser_main.cc(285)] Handling shutdown for '
  252. 'signal 15.' + """
  253. [----------] 4 tests from SomeOtherTest
  254. [ RUN ] SomeOtherTest.SwitchTypes
  255. %(switch_errors)s
  256. [ FAILED ] SomeOtherTest.SwitchTypes (40 ms)
  257. [ RUN ] SomeOtherTest.Foo
  258. [ OK ] SomeOtherTest.Foo (20 ms)
  259. [ RUN ] SomeOtherTest.FAILS_Bar
  260. Some error message for a failing test.
  261. [ FAILED ] SomeOtherTest.FAILS_Bar (40 ms)
  262. [ RUN ] SomeOtherTest.FAILS_ThisTestTimesOut
  263. """ % {
  264. 'switch_errors': SWITCH_ERRORS
  265. } +
  266. '[0521/041343:ERROR:test_launcher.cc(384)] Test timeout (5000 ms) exceeded '
  267. 'for SomeOtherTest.FAILS_ThisTestTimesOut' + """
  268. [ RUN ] SomeOtherTest.FLAKY_Baz
  269. Some error message for a flaky test.
  270. [ FAILED ] SomeOtherTest.FLAKY_Baz (40 ms)
  271. [----------] 2 tests from SomeOtherTest (60 ms total)
  272. [----------] Global test environment tear-down
  273. [==========] 7 tests from 3 test cases ran. (3750 ms total)
  274. [ PASSED ] 5 tests.
  275. [ FAILED ] 2 test, listed below:
  276. [ FAILED ] SomeOtherTest.SwitchTypes
  277. [ FAILED ] SomeOtherTest.FAILS_ThisTestTimesOut
  278. 1 FAILED TEST
  279. YOU HAVE 10 DISABLED TESTS
  280. YOU HAVE 2 FLAKY TESTS
  281. """)
  282. TEST_DATA_SHARD_1 = (
  283. """Note: This is test shard 13 of 30.
  284. [==========] Running 5 tests from 2 test cases.
  285. [----------] Global test environment set-up.
  286. [----------] 4 tests from NavigationControllerTest
  287. [ RUN ] NavigationControllerTest.Defaults
  288. [ OK ] NavigationControllerTest.Defaults (48 ms)
  289. [ RUN ] NavigationControllerTest.Reload
  290. %(reload_errors)s
  291. [ FAILED ] NavigationControllerTest.Reload (2 ms)
  292. [ RUN ] NavigationControllerTest.Reload_GeneratesNewPage
  293. [ OK ] NavigationControllerTest.Reload_GeneratesNewPage (22 ms)
  294. [ RUN ] NavigationControllerTest/SpdyNetworkTransTest.Constructor/0
  295. %(spdy_errors)s
  296. """ % {
  297. 'reload_errors': RELOAD_ERRORS,
  298. 'spdy_errors': SPDY_ERRORS
  299. } + '[ FAILED ] NavigationControllerTest/SpdyNetworkTransTest.Constructor'
  300. '/0 (2 ms)' + """
  301. [----------] 4 tests from NavigationControllerTest (74 ms total)
  302. YOU HAVE 2 FLAKY TESTS
  303. [----------] 1 test from MoreBadTest
  304. [ RUN ] MoreBadTest.TimesOutAndFails
  305. %(morebad_errors)s
  306. """ % {
  307. 'morebad_errors': MOREBAD_ERRORS
  308. } +
  309. '[0531/042642:ERROR:/b/slave/chromium-rel-mac-builder/build/src/chrome/test'
  310. '/test_launcher/out_of_proc_test_runner.cc(79)] Test timeout (30000 ms) '
  311. 'exceeded for MoreBadTest.TimesOutAndFails' + """
  312. Handling SIGTERM.
  313. Successfully wrote to shutdown pipe, resetting signal handler.
  314. [ FAILED ] MoreBadTest.TimesOutAndFails (31000 ms)
  315. [----------] Global test environment tear-down
  316. [==========] 5 tests from 2 test cases ran. (3750 ms total)
  317. [ PASSED ] 3 tests.
  318. [ FAILED ] 2 tests, listed below:
  319. [ FAILED ] NavigationControllerTest.Reload
  320. [ FAILED ] NavigationControllerTest/SpdyNetworkTransTest.Constructor/0
  321. 1 FAILED TEST
  322. YOU HAVE 10 DISABLED TESTS
  323. YOU HAVE 2 FLAKY TESTS
  324. """)
  325. TEST_DATA_SHARD_EXIT = 'program finished with exit code '
  326. TEST_DATA_CRASH_SHARD = """Note: This is test shard 5 of 5.
  327. [==========] Running 7 tests from 3 test cases.
  328. [----------] Global test environment set-up.
  329. [----------] 1 test from HunspellTest
  330. [ RUN ] HunspellTest.Crashes
  331. Oops, this test crashed!"""
  332. TEST_DATA_NESTED_RUNS = (
  333. """
  334. [ 1/3] 1.0s Foo.Bar (45.5s)
  335. Note: Google Test filter = Foo.Bar
  336. [==========] Running 1 test from 1 test case.
  337. [----------] Global test environment set-up.
  338. [----------] 1 test from Foo, where TypeParam =
  339. [ RUN ] Foo.Bar
  340. """ +
  341. '[0725/050653:ERROR:test_launcher.cc(380)] Test timeout (45000 ms) exceeded '
  342. 'for Foo.Bar' + """
  343. Starting tests...
  344. IMPORTANT DEBUGGING NOTE: each test is run inside its own process.
  345. For debugging a test inside a debugger, use the
  346. --gtest_filter=<your_test_name> flag along with either
  347. --single_process (to run all tests in one launcher/browser process) or
  348. --single-process (to do the above, and also run Chrome in single-
  349. process mode).
  350. 1 test run
  351. 1 test failed (0 ignored)
  352. Failing tests:
  353. Foo.Bar
  354. [ 2/2] 2.00s Foo.Pass (1.0s)""")
  355. # Data generated with run_test_case.py
  356. TEST_DATA_RUN_TEST_CASE_FAIL = """
  357. [ 6/422] 7.45s SUIDSandboxUITest.testSUIDSandboxEnabled (1.49s) - retry #2
  358. [ RUN ] SUIDSandboxUITest.testSUIDSandboxEnabled
  359. [ FAILED ] SUIDSandboxUITest.testSUIDSandboxEnabled (771 ms)
  360. [ 8/422] 7.76s PrintPreviewWebUITest.SourceIsPDFShowFitToPageOption (1.67s)
  361. """
  362. TEST_DATA_RUN_TEST_CASE_TIMEOUT = """
  363. [ 6/422] 7.45s SUIDSandboxUITest.testSUIDSandboxEnabled (1.49s) - retry #2
  364. [ RUN ] SUIDSandboxUITest.testSUIDSandboxEnabled
  365. (junk)
  366. [ 8/422] 7.76s PrintPreviewWebUITest.SourceIsPDFShowFitToPageOption (1.67s)
  367. """
  368. # Data generated by swarming.py
  369. TEST_DATA_SWARM_TEST_FAIL = """
  370. ================================================================
  371. Begin output from shard index 0 (machine tag: swarm12.c, id: swarm12)
  372. ================================================================
  373. [==========] Running 2 tests from linux_swarm_trigg-8-base_unittests test run.
  374. Starting tests (using 2 parallel jobs)...
  375. IMPORTANT DEBUGGING NOTE: batches of tests are run inside their
  376. own process. For debugging a test inside a debugger, use the
  377. --gtest_filter=<your_test_name> flag along with
  378. --single-process-tests.
  379. [1/1242] HistogramDeathTest.BadRangesTest (62 ms)
  380. [2/1242] OutOfMemoryDeathTest.New (22 ms)
  381. [1242/1242] ThreadIdNameManagerTest.ThreadNameInterning (0 ms)
  382. Retrying 1 test (retry #1)
  383. [ RUN ] PickleTest.EncodeDecode
  384. ../../base/pickle_unittest.cc:69: Failure
  385. Value of: false
  386. Actual: false
  387. Expected: true
  388. [ FAILED ] PickleTest.EncodeDecode (0 ms)
  389. [1243/1243] PickleTest.EncodeDecode (0 ms)
  390. Retrying 1 test (retry #2)
  391. [ RUN ] PickleTest.EncodeDecode
  392. ../../base/pickle_unittest.cc:69: Failure
  393. Value of: false
  394. Actual: false
  395. Expected: true
  396. [ FAILED ] PickleTest.EncodeDecode (1 ms)
  397. [1244/1244] PickleTest.EncodeDecode (1 ms)
  398. Retrying 1 test (retry #3)
  399. [ RUN ] PickleTest.EncodeDecode
  400. ../../base/pickle_unittest.cc:69: Failure
  401. Value of: false
  402. Actual: false
  403. Expected: true
  404. [ FAILED ] PickleTest.EncodeDecode (0 ms)
  405. [1245/1245] PickleTest.EncodeDecode (0 ms)
  406. 1245 tests run
  407. 1 test failed:
  408. PickleTest.EncodeDecode
  409. Summary of all itest iterations:
  410. 1 test failed:
  411. PickleTest.EncodeDecode
  412. End of the summary.
  413. Tests took 31 seconds.
  414. ================================================================
  415. End output from shard index 0 (machine tag: swarm12.c, id: swarm12). Return 1
  416. ================================================================
  417. """
  418. # pylint: enable=line-too-long
  419. class TestGTestLogParserTests(unittest.TestCase):
  420. def testGTestLogParserNoSharing(self):
  421. # Tests for log parsing without sharding.
  422. parser = gtest_utils.GTestLogParser()
  423. for line in TEST_DATA.splitlines():
  424. parser.ProcessLine(line)
  425. parser.Finalize()
  426. self.assertEqual(0, len(parser.ParsingErrors()))
  427. self.assertFalse(parser.RunningTests())
  428. self.assertEqual(sorted(FAILURES), sorted(parser.FailedTests()))
  429. self.assertEqual(
  430. sorted(FAILURES + FAILS_FAILURES),
  431. sorted(parser.FailedTests(include_fails=True)))
  432. self.assertEqual(
  433. sorted(FAILURES + FLAKY_FAILURES),
  434. sorted(parser.FailedTests(include_flaky=True)))
  435. self.assertEqual(
  436. sorted(FAILURES + FAILS_FAILURES + FLAKY_FAILURES),
  437. sorted(parser.FailedTests(include_fails=True, include_flaky=True)))
  438. self.assertEqual(10, parser.DisabledTests())
  439. self.assertEqual(2, parser.FlakyTests())
  440. test_name = 'NavigationControllerTest.Reload'
  441. self.assertEqual('\n'.join(['%s: ' % test_name, RELOAD_ERRORS]),
  442. '\n'.join(parser.FailureDescription(test_name)))
  443. self.assertEqual(['FAILURE'], parser.TriesForTest(test_name))
  444. test_name = 'NavigationControllerTest/SpdyNetworkTransTest.Constructor/0'
  445. self.assertEqual('\n'.join(['%s: ' % test_name, SPDY_ERRORS]),
  446. '\n'.join(parser.FailureDescription(test_name)))
  447. self.assertEqual(['FAILURE'], parser.TriesForTest(test_name))
  448. test_name = 'SomeOtherTest.SwitchTypes'
  449. self.assertEqual('\n'.join(['%s: ' % test_name, SWITCH_ERRORS]),
  450. '\n'.join(parser.FailureDescription(test_name)))
  451. self.assertEqual(['FAILURE'], parser.TriesForTest(test_name))
  452. test_name = 'BadTest.TimesOut'
  453. self.assertEqual(
  454. '\n'.join(['%s: ' % test_name, TIMEOUT_ERRORS, TIMEOUT_MESSAGE]),
  455. '\n'.join(parser.FailureDescription(test_name)))
  456. self.assertEqual(['TIMEOUT'], parser.TriesForTest(test_name))
  457. test_name = 'MoreBadTest.TimesOutAndFails'
  458. self.assertEqual(
  459. '\n'.join(['%s: ' % test_name, MOREBAD_ERRORS, TIMEOUT_MESSAGE]),
  460. '\n'.join(parser.FailureDescription(test_name)))
  461. self.assertEqual(['TIMEOUT'], parser.TriesForTest(test_name))
  462. self.assertEqual(['SUCCESS'], parser.TriesForTest('SomeOtherTest.Foo'))
  463. # Same unit tests (when applicable) using ResultCollection
  464. collection = parser.GetResultCollection()
  465. self.assertEqual(
  466. sorted(FAILURES + FAILS_FAILURES + FLAKY_FAILURES),
  467. sorted(collection.never_expected_tests()))
  468. self.assertEqual(len(collection.test_results), 12)
  469. # To know that each condition branch in for loop is covered.
  470. cover_set = set()
  471. for test_result in collection.test_results:
  472. name = test_result.name
  473. if name == 'NavigationControllerTest.Reload':
  474. cover_set.add(name)
  475. self.assertEqual('\n'.join([RELOAD_ERRORS]), test_result.test_log)
  476. self.assertEqual(TestStatus.FAIL, test_result.status)
  477. self.assertEqual(2, test_result.duration)
  478. if name == 'NavigationControllerTest/SpdyNetworkTransTest.Constructor/0':
  479. cover_set.add(name)
  480. self.assertEqual('\n'.join([SPDY_ERRORS]), test_result.test_log)
  481. self.assertEqual(TestStatus.FAIL, test_result.status)
  482. self.assertEqual(2, test_result.duration)
  483. if name == 'SomeOtherTest.SwitchTypes':
  484. cover_set.add(name)
  485. self.assertEqual('\n'.join([SWITCH_ERRORS]), test_result.test_log)
  486. self.assertEqual(TestStatus.FAIL, test_result.status)
  487. self.assertEqual(40, test_result.duration)
  488. if name == 'BadTest.TimesOut':
  489. cover_set.add(name)
  490. self.assertEqual('\n'.join([TIMEOUT_ERRORS, TIMEOUT_MESSAGE]),
  491. test_result.test_log)
  492. self.assertEqual(TestStatus.ABORT, test_result.status)
  493. self.assertEqual(None, test_result.duration)
  494. if name == 'MoreBadTest.TimesOutAndFails':
  495. cover_set.add(name)
  496. self.assertEqual('\n'.join([MOREBAD_ERRORS, TIMEOUT_MESSAGE]),
  497. test_result.test_log)
  498. self.assertEqual(TestStatus.ABORT, test_result.status)
  499. self.assertEqual(None, test_result.duration)
  500. if name == 'SomeOtherTest.Foo':
  501. cover_set.add(name)
  502. self.assertEqual('', test_result.test_log)
  503. self.assertEqual(TestStatus.PASS, test_result.status)
  504. self.assertEqual(20, test_result.duration)
  505. test_list = [
  506. 'BadTest.TimesOut', 'MoreBadTest.TimesOutAndFails',
  507. 'NavigationControllerTest.Reload',
  508. 'NavigationControllerTest/SpdyNetworkTransTest.Constructor/0',
  509. 'SomeOtherTest.Foo', 'SomeOtherTest.SwitchTypes'
  510. ]
  511. self.assertEqual(sorted(test_list), sorted(cover_set))
  512. parser = gtest_utils.GTestLogParser()
  513. for line in TEST_DATA_CRASH.splitlines():
  514. parser.ProcessLine(line)
  515. parser.Finalize()
  516. self.assertEqual(0, len(parser.ParsingErrors()))
  517. self.assertTrue(parser.RunningTests())
  518. self.assertEqual(['HunspellTest.Crashes'], parser.FailedTests())
  519. self.assertEqual(0, parser.DisabledTests())
  520. self.assertEqual(0, parser.FlakyTests())
  521. test_name = 'HunspellTest.Crashes'
  522. expected_log_lines = [
  523. 'Did not complete.',
  524. 'Potential test logs from crash until the end of test program:'
  525. ] + CRASH_MESSAGE
  526. self.assertEqual('\n'.join(['%s: ' % test_name] + expected_log_lines),
  527. '\n'.join(parser.FailureDescription(test_name)))
  528. self.assertEqual(['UNKNOWN'], parser.TriesForTest(test_name))
  529. collection = parser.GetResultCollection()
  530. self.assertEqual(
  531. set(['HunspellTest.Crashes']), collection.unexpected_tests())
  532. for result in collection.test_results:
  533. covered = False
  534. if result.name == 'HunspellTest.Crashes':
  535. covered = True
  536. self.assertEqual('\n'.join(expected_log_lines), result.test_log)
  537. self.assertEqual(TestStatus.CRASH, result.status)
  538. self.assertTrue(covered)
  539. def testGTestLogParserSharing(self):
  540. # Same tests for log parsing with sharding_supervisor.
  541. parser = gtest_utils.GTestLogParser()
  542. test_data_shard = TEST_DATA_SHARD_0 + TEST_DATA_SHARD_1
  543. for line in test_data_shard.splitlines():
  544. parser.ProcessLine(line)
  545. parser.ProcessLine(TEST_DATA_SHARD_EXIT + '2')
  546. parser.Finalize()
  547. self.assertEqual(0, len(parser.ParsingErrors()))
  548. self.assertFalse(parser.RunningTests())
  549. self.assertEqual(sorted(FAILURES), sorted(parser.FailedTests()))
  550. self.assertEqual(
  551. sorted(FAILURES + FAILS_FAILURES),
  552. sorted(parser.FailedTests(include_fails=True)))
  553. self.assertEqual(
  554. sorted(FAILURES + FLAKY_FAILURES),
  555. sorted(parser.FailedTests(include_flaky=True)))
  556. self.assertEqual(
  557. sorted(FAILURES + FAILS_FAILURES + FLAKY_FAILURES),
  558. sorted(parser.FailedTests(include_fails=True, include_flaky=True)))
  559. self.assertEqual(10, parser.DisabledTests())
  560. self.assertEqual(2, parser.FlakyTests())
  561. test_name = 'NavigationControllerTest.Reload'
  562. self.assertEqual('\n'.join(['%s: ' % test_name, RELOAD_ERRORS]),
  563. '\n'.join(parser.FailureDescription(test_name)))
  564. self.assertEqual(['FAILURE'], parser.TriesForTest(test_name))
  565. test_name = ('NavigationControllerTest/SpdyNetworkTransTest.Constructor/0')
  566. self.assertEqual('\n'.join(['%s: ' % test_name, SPDY_ERRORS]),
  567. '\n'.join(parser.FailureDescription(test_name)))
  568. self.assertEqual(['FAILURE'], parser.TriesForTest(test_name))
  569. test_name = 'SomeOtherTest.SwitchTypes'
  570. self.assertEqual('\n'.join(['%s: ' % test_name, SWITCH_ERRORS]),
  571. '\n'.join(parser.FailureDescription(test_name)))
  572. self.assertEqual(['FAILURE'], parser.TriesForTest(test_name))
  573. test_name = 'BadTest.TimesOut'
  574. self.assertEqual(
  575. '\n'.join(['%s: ' % test_name, TIMEOUT_ERRORS, TIMEOUT_MESSAGE]),
  576. '\n'.join(parser.FailureDescription(test_name)))
  577. self.assertEqual(['TIMEOUT'], parser.TriesForTest(test_name))
  578. test_name = 'MoreBadTest.TimesOutAndFails'
  579. self.assertEqual(
  580. '\n'.join(['%s: ' % test_name, MOREBAD_ERRORS, TIMEOUT_MESSAGE]),
  581. '\n'.join(parser.FailureDescription(test_name)))
  582. self.assertEqual(['TIMEOUT'], parser.TriesForTest(test_name))
  583. self.assertEqual(['SUCCESS'], parser.TriesForTest('SomeOtherTest.Foo'))
  584. # Same unit tests (when applicable) using ResultCollection
  585. collection = parser.GetResultCollection()
  586. self.assertEqual(
  587. sorted(FAILURES + FAILS_FAILURES + FLAKY_FAILURES),
  588. sorted(collection.never_expected_tests()))
  589. self.assertEqual(len(collection.test_results), 12)
  590. # To know that each condition branch in for loop is covered.
  591. cover_set = set()
  592. for test_result in collection.test_results:
  593. name = test_result.name
  594. if name == 'NavigationControllerTest.Reload':
  595. cover_set.add(name)
  596. self.assertEqual('\n'.join([RELOAD_ERRORS]), test_result.test_log)
  597. self.assertEqual(TestStatus.FAIL, test_result.status)
  598. self.assertEqual(2, test_result.duration)
  599. if name == 'NavigationControllerTest/SpdyNetworkTransTest.Constructor/0':
  600. cover_set.add(name)
  601. self.assertEqual('\n'.join([SPDY_ERRORS]), test_result.test_log)
  602. self.assertEqual(TestStatus.FAIL, test_result.status)
  603. self.assertEqual(2, test_result.duration)
  604. if name == 'SomeOtherTest.SwitchTypes':
  605. cover_set.add(name)
  606. self.assertEqual('\n'.join([SWITCH_ERRORS]), test_result.test_log)
  607. self.assertEqual(TestStatus.FAIL, test_result.status)
  608. self.assertEqual(40, test_result.duration)
  609. if name == 'BadTest.TimesOut':
  610. cover_set.add(name)
  611. self.assertEqual('\n'.join([TIMEOUT_ERRORS, TIMEOUT_MESSAGE]),
  612. test_result.test_log)
  613. self.assertEqual(TestStatus.ABORT, test_result.status)
  614. self.assertEqual(None, test_result.duration)
  615. if name == 'MoreBadTest.TimesOutAndFails':
  616. cover_set.add(name)
  617. self.assertEqual('\n'.join([MOREBAD_ERRORS, TIMEOUT_MESSAGE]),
  618. test_result.test_log)
  619. self.assertEqual(TestStatus.ABORT, test_result.status)
  620. self.assertEqual(None, test_result.duration)
  621. if name == 'SomeOtherTest.Foo':
  622. cover_set.add(name)
  623. self.assertEqual('', test_result.test_log)
  624. self.assertEqual(TestStatus.PASS, test_result.status)
  625. test_list = [
  626. 'BadTest.TimesOut', 'MoreBadTest.TimesOutAndFails',
  627. 'NavigationControllerTest.Reload',
  628. 'NavigationControllerTest/SpdyNetworkTransTest.Constructor/0',
  629. 'SomeOtherTest.Foo', 'SomeOtherTest.SwitchTypes'
  630. ]
  631. self.assertEqual(sorted(test_list), sorted(cover_set))
  632. parser = gtest_utils.GTestLogParser()
  633. for line in TEST_DATA_CRASH.splitlines():
  634. parser.ProcessLine(line)
  635. parser.Finalize()
  636. self.assertEqual(0, len(parser.ParsingErrors()))
  637. self.assertTrue(parser.RunningTests())
  638. self.assertEqual(['HunspellTest.Crashes'], parser.FailedTests())
  639. self.assertEqual(0, parser.DisabledTests())
  640. self.assertEqual(0, parser.FlakyTests())
  641. test_name = 'HunspellTest.Crashes'
  642. expected_log_lines = [
  643. 'Did not complete.',
  644. 'Potential test logs from crash until the end of test program:'
  645. ] + CRASH_MESSAGE
  646. self.assertEqual('\n'.join(['%s: ' % test_name] + expected_log_lines),
  647. '\n'.join(parser.FailureDescription(test_name)))
  648. self.assertEqual(['UNKNOWN'], parser.TriesForTest(test_name))
  649. collection = parser.GetResultCollection()
  650. self.assertEqual(
  651. set(['HunspellTest.Crashes']), collection.unexpected_tests())
  652. for result in collection.test_results:
  653. covered = False
  654. if result.name == 'HunspellTest.Crashes':
  655. covered = True
  656. self.assertEqual('\n'.join(expected_log_lines), result.test_log)
  657. self.assertEqual(TestStatus.CRASH, result.status)
  658. self.assertTrue(covered)
  659. def testGTestLogParserMixedStdout(self):
  660. parser = gtest_utils.GTestLogParser()
  661. for line in TEST_DATA_MIXED_STDOUT.splitlines():
  662. parser.ProcessLine(line)
  663. parser.Finalize()
  664. self.assertEqual([], parser.ParsingErrors())
  665. self.assertEqual(['Crash.Test'], parser.RunningTests())
  666. self.assertEqual(['TestFix.TestCase', 'Crash.Test'], parser.FailedTests())
  667. self.assertEqual(0, parser.DisabledTests())
  668. self.assertEqual(0, parser.FlakyTests())
  669. self.assertEqual(['UNKNOWN'], parser.TriesForTest('Crash.Test'))
  670. self.assertEqual(['TIMEOUT'], parser.TriesForTest('TestFix.TestCase'))
  671. self.assertEqual(['SUCCESS'],
  672. parser.TriesForTest(
  673. 'WebSocketHandshakeHandlerSpdy3Test.RequestResponse'))
  674. # Same unit tests (when applicable) using ResultCollection
  675. collection = parser.GetResultCollection()
  676. self.assertEqual(
  677. sorted(['TestFix.TestCase', 'Crash.Test']),
  678. sorted(collection.never_expected_tests()))
  679. # To know that each condition branch in for loop is covered.
  680. cover_set = set()
  681. for test_result in collection.test_results:
  682. name = test_result.name
  683. if name == 'Crash.Test':
  684. cover_set.add(name)
  685. self.assertEqual(TestStatus.CRASH, test_result.status)
  686. if name == 'TestFix.TestCase':
  687. cover_set.add(name)
  688. self.assertEqual(TestStatus.ABORT, test_result.status)
  689. if name == 'WebSocketHandshakeHandlerSpdy3Test.RequestResponse':
  690. cover_set.add(name)
  691. self.assertEqual(TestStatus.PASS, test_result.status)
  692. self.assertEqual(1, test_result.duration)
  693. test_list = [
  694. 'Crash.Test', 'TestFix.TestCase',
  695. 'WebSocketHandshakeHandlerSpdy3Test.RequestResponse'
  696. ]
  697. self.assertEqual(test_list, sorted(cover_set))
  698. def testGtestLogParserSkipped(self):
  699. parser = gtest_utils.GTestLogParser()
  700. for line in TEST_DATA_SKIPPED.splitlines():
  701. parser.ProcessLine(line)
  702. parser.Finalize()
  703. self.assertEqual([], parser.ParsingErrors())
  704. self.assertEqual([], parser.RunningTests())
  705. self.assertEqual([], parser.FailedTests())
  706. self.assertEqual(['ProcessReaderLinux.AbortMessage'], parser.SkippedTests())
  707. self.assertEqual(0, parser.DisabledTests())
  708. self.assertEqual(0, parser.FlakyTests())
  709. self.assertEqual(['SKIPPED'],
  710. parser.TriesForTest('ProcessReaderLinux.AbortMessage'))
  711. # Same unit tests (when applicable) using ResultCollection
  712. collection = parser.GetResultCollection()
  713. self.assertEqual(['ProcessReaderLinux.AbortMessage'],
  714. sorted(
  715. collection.tests_by_expression(lambda tr: tr.status ==
  716. TestStatus.SKIP)))
  717. self.assertEqual([], sorted(collection.unexpected_tests()))
  718. covered = False
  719. for test_result in collection.test_results:
  720. if test_result.name == 'ProcessReaderLinux.AbortMessage':
  721. covered = True
  722. self.assertEqual(TestStatus.SKIP, test_result.status)
  723. self.assertTrue(covered)
  724. def testRunTestCaseFail(self):
  725. parser = gtest_utils.GTestLogParser()
  726. for line in TEST_DATA_RUN_TEST_CASE_FAIL.splitlines():
  727. parser.ProcessLine(line)
  728. parser.Finalize()
  729. self.assertEqual(0, len(parser.ParsingErrors()))
  730. self.assertEqual([], parser.RunningTests())
  731. self.assertEqual(['SUIDSandboxUITest.testSUIDSandboxEnabled'],
  732. parser.FailedTests())
  733. self.assertEqual(
  734. ['SUIDSandboxUITest.testSUIDSandboxEnabled: '],
  735. parser.FailureDescription('SUIDSandboxUITest.testSUIDSandboxEnabled'))
  736. self.assertEqual(
  737. ['FAILURE'],
  738. parser.TriesForTest('SUIDSandboxUITest.testSUIDSandboxEnabled'))
  739. # Same unit tests (when applicable) using ResultCollection
  740. collection = parser.GetResultCollection()
  741. self.assertEqual(['SUIDSandboxUITest.testSUIDSandboxEnabled'],
  742. sorted(collection.failed_tests()))
  743. covered = False
  744. for test_result in collection.test_results:
  745. if test_result.name == 'SUIDSandboxUITest.testSUIDSandboxEnabled':
  746. covered = True
  747. self.assertEqual(TestStatus.FAIL, test_result.status)
  748. self.assertEqual('', test_result.test_log)
  749. self.assertEqual(771, test_result.duration)
  750. self.assertTrue(covered)
  751. def testRunTestCaseTimeout(self):
  752. parser = gtest_utils.GTestLogParser()
  753. for line in TEST_DATA_RUN_TEST_CASE_TIMEOUT.splitlines():
  754. parser.ProcessLine(line)
  755. parser.Finalize()
  756. self.assertEqual(0, len(parser.ParsingErrors()))
  757. self.assertEqual([], parser.RunningTests())
  758. self.assertEqual(['SUIDSandboxUITest.testSUIDSandboxEnabled'],
  759. parser.FailedTests())
  760. self.assertEqual(
  761. ['SUIDSandboxUITest.testSUIDSandboxEnabled: ', '(junk)'],
  762. parser.FailureDescription('SUIDSandboxUITest.testSUIDSandboxEnabled'))
  763. self.assertEqual(
  764. ['TIMEOUT'],
  765. parser.TriesForTest('SUIDSandboxUITest.testSUIDSandboxEnabled'))
  766. # Same unit tests (when applicable) using ResultCollection
  767. collection = parser.GetResultCollection()
  768. self.assertEqual(['SUIDSandboxUITest.testSUIDSandboxEnabled'],
  769. sorted(collection.never_expected_tests()))
  770. covered = False
  771. for test_result in collection.test_results:
  772. if test_result.name == 'SUIDSandboxUITest.testSUIDSandboxEnabled':
  773. covered = True
  774. self.assertEqual(TestStatus.ABORT, test_result.status)
  775. self.assertEqual('(junk)', test_result.test_log)
  776. self.assertEqual(None, test_result.duration)
  777. self.assertTrue(covered)
  778. def testRunTestCaseParseSwarm(self):
  779. parser = gtest_utils.GTestLogParser()
  780. for line in TEST_DATA_SWARM_TEST_FAIL.splitlines():
  781. parser.ProcessLine(line)
  782. parser.Finalize()
  783. self.assertEqual(0, len(parser.ParsingErrors()))
  784. self.assertEqual([], parser.RunningTests())
  785. self.assertEqual(['PickleTest.EncodeDecode'], parser.FailedTests())
  786. log_lines = [
  787. 'PickleTest.EncodeDecode: ',
  788. '../../base/pickle_unittest.cc:69: Failure',
  789. 'Value of: false',
  790. ' Actual: false',
  791. 'Expected: true',
  792. ]
  793. self.assertEqual(log_lines,
  794. parser.FailureDescription('PickleTest.EncodeDecode'))
  795. self.assertEqual(['FAILURE'],
  796. parser.TriesForTest('PickleTest.EncodeDecode'))
  797. # Same unit tests (when applicable) using ResultCollection
  798. collection = parser.GetResultCollection()
  799. self.assertEqual(['PickleTest.EncodeDecode'],
  800. sorted(collection.never_expected_tests()))
  801. covered_count = 0
  802. for test_result in collection.test_results:
  803. if test_result.name == 'PickleTest.EncodeDecode':
  804. covered_count += 1
  805. self.assertEqual(TestStatus.FAIL, test_result.status)
  806. self.assertEqual('\n'.join(log_lines[1:]), test_result.test_log)
  807. self.assertEqual(3, covered_count)
  808. def testNestedGtests(self):
  809. parser = gtest_utils.GTestLogParser()
  810. for line in TEST_DATA_NESTED_RUNS.splitlines():
  811. parser.ProcessLine(line)
  812. parser.Finalize()
  813. self.assertEqual(['Foo.Bar'], parser.FailedTests(True, True))
  814. # Same unit tests (when applicable) using ResultCollection
  815. collection = parser.GetResultCollection()
  816. self.assertEqual(['Foo.Bar'], sorted(collection.never_expected_tests()))
  817. covered = False
  818. for test_result in collection.test_results:
  819. if test_result.name == 'Foo.Bar':
  820. covered = True
  821. self.assertEqual(TestStatus.ABORT, test_result.status)
  822. self.assertTrue(covered)
  823. if __name__ == '__main__':
  824. unittest.main()