expectations_unittest.py 32 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826
  1. #!/usr/bin/env vpython3
  2. # Copyright 2020 The Chromium Authors. All rights reserved.
  3. # Use of this source code is governed by a BSD-style license that can be
  4. # found in the LICENSE file.
  5. from __future__ import print_function
  6. import datetime
  7. import os
  8. import sys
  9. import tempfile
  10. import unittest
  11. if sys.version_info[0] == 2:
  12. import mock
  13. else:
  14. import unittest.mock as mock
  15. from pyfakefs import fake_filesystem_unittest
  16. from unexpected_passes_common import data_types
  17. from unexpected_passes_common import expectations
  18. from unexpected_passes_common import unittest_utils as uu
  19. FAKE_EXPECTATION_FILE_CONTENTS = """\
  20. # tags: [ win linux ]
  21. # results: [ Failure RetryOnFailure Skip Pass ]
  22. crbug.com/1234 [ win ] foo/test [ Failure ]
  23. crbug.com/5678 crbug.com/6789 [ win ] foo/another/test [ RetryOnFailure ]
  24. [ linux ] foo/test [ Failure ]
  25. crbug.com/2345 [ linux ] bar/* [ RetryOnFailure ]
  26. crbug.com/3456 [ linux ] some/bad/test [ Skip ]
  27. crbug.com/4567 [ linux ] some/good/test [ Pass ]
  28. """
  29. SECONDARY_FAKE_EXPECTATION_FILE_CONTENTS = """\
  30. # tags: [ mac ]
  31. # results: [ Failure ]
  32. crbug.com/4567 [ mac ] foo/test [ Failure ]
  33. """
  34. FAKE_EXPECTATION_FILE_CONTENTS_WITH_TYPO = """\
  35. # tags: [ win linux ]
  36. # results: [ Failure RetryOnFailure Skip ]
  37. crbug.com/1234 [ wine ] foo/test [ Failure ]
  38. [ linux ] foo/test [ Failure ]
  39. crbug.com/2345 [ linux ] bar/* [ RetryOnFailure ]
  40. crbug.com/3456 [ linux ] some/bad/test [ Skip ]
  41. """
  42. class CreateTestExpectationMapUnittest(unittest.TestCase):
  43. def setUp(self) -> None:
  44. self.instance = expectations.Expectations()
  45. self._expectation_content = {}
  46. self._content_patcher = mock.patch.object(
  47. self.instance, '_GetNonRecentExpectationContent')
  48. self._content_mock = self._content_patcher.start()
  49. self.addCleanup(self._content_patcher.stop)
  50. def SideEffect(filepath, _):
  51. return self._expectation_content[filepath]
  52. self._content_mock.side_effect = SideEffect
  53. def testExclusiveOr(self) -> None:
  54. """Tests that only one input can be specified."""
  55. with self.assertRaises(AssertionError):
  56. self.instance.CreateTestExpectationMap(None, None, 0)
  57. with self.assertRaises(AssertionError):
  58. self.instance.CreateTestExpectationMap('foo', ['bar'], 0)
  59. def testExpectationFile(self) -> None:
  60. """Tests reading expectations from an expectation file."""
  61. filename = '/tmp/foo'
  62. self._expectation_content[filename] = FAKE_EXPECTATION_FILE_CONTENTS
  63. expectation_map = self.instance.CreateTestExpectationMap(filename, None, 0)
  64. # Skip expectations should be omitted, but everything else should be
  65. # present.
  66. # yapf: disable
  67. expected_expectation_map = {
  68. filename: {
  69. data_types.Expectation(
  70. 'foo/test', ['win'], ['Failure'], 'crbug.com/1234'): {},
  71. data_types.Expectation(
  72. 'foo/another/test', ['win'], ['RetryOnFailure'],
  73. 'crbug.com/5678 crbug.com/6789'): {},
  74. data_types.Expectation('foo/test', ['linux'], ['Failure']): {},
  75. data_types.Expectation(
  76. 'bar/*', ['linux'], ['RetryOnFailure'], 'crbug.com/2345'): {},
  77. },
  78. }
  79. # yapf: enable
  80. self.assertEqual(expectation_map, expected_expectation_map)
  81. self.assertIsInstance(expectation_map, data_types.TestExpectationMap)
  82. def testMultipleExpectationFiles(self) -> None:
  83. """Tests reading expectations from multiple files."""
  84. filename1 = '/tmp/foo'
  85. filename2 = '/tmp/bar'
  86. expectation_files = [filename1, filename2]
  87. self._expectation_content[filename1] = FAKE_EXPECTATION_FILE_CONTENTS
  88. self._expectation_content[
  89. filename2] = SECONDARY_FAKE_EXPECTATION_FILE_CONTENTS
  90. expectation_map = self.instance.CreateTestExpectationMap(
  91. expectation_files, None, 0)
  92. # yapf: disable
  93. expected_expectation_map = {
  94. expectation_files[0]: {
  95. data_types.Expectation(
  96. 'foo/test', ['win'], ['Failure'], 'crbug.com/1234'): {},
  97. data_types.Expectation(
  98. 'foo/another/test', ['win'], ['RetryOnFailure'],
  99. 'crbug.com/5678 crbug.com/6789'): {},
  100. data_types.Expectation('foo/test', ['linux'], ['Failure']): {},
  101. data_types.Expectation(
  102. 'bar/*', ['linux'], ['RetryOnFailure'], 'crbug.com/2345'): {},
  103. },
  104. expectation_files[1]: {
  105. data_types.Expectation(
  106. 'foo/test', ['mac'], ['Failure'], 'crbug.com/4567'): {},
  107. }
  108. }
  109. # yapf: enable
  110. self.assertEqual(expectation_map, expected_expectation_map)
  111. self.assertIsInstance(expectation_map, data_types.TestExpectationMap)
  112. def testIndividualTests(self) -> None:
  113. """Tests reading expectations from a list of tests."""
  114. expectation_map = self.instance.CreateTestExpectationMap(
  115. None, ['foo/test', 'bar/*'], 0)
  116. expected_expectation_map = {
  117. '': {
  118. data_types.Expectation('foo/test', [], ['RetryOnFailure']): {},
  119. data_types.Expectation('bar/*', [], ['RetryOnFailure']): {},
  120. },
  121. }
  122. self.assertEqual(expectation_map, expected_expectation_map)
  123. self.assertIsInstance(expectation_map, data_types.TestExpectationMap)
  124. class GetNonRecentExpectationContentUnittest(unittest.TestCase):
  125. def setUp(self) -> None:
  126. self.instance = uu.CreateGenericExpectations()
  127. self._output_patcher = mock.patch(
  128. 'unexpected_passes_common.expectations.subprocess.check_output')
  129. self._output_mock = self._output_patcher.start()
  130. self.addCleanup(self._output_patcher.stop)
  131. def testBasic(self) -> None:
  132. """Tests that only expectations that are old enough are kept."""
  133. today_date = datetime.date.today()
  134. yesterday_date = today_date - datetime.timedelta(days=1)
  135. older_date = today_date - datetime.timedelta(days=2)
  136. today_str = today_date.isoformat()
  137. yesterday_str = yesterday_date.isoformat()
  138. older_str = older_date.isoformat()
  139. # pylint: disable=line-too-long
  140. blame_output = """\
  141. 5f03bc04975c04 (Some R. Author {today_date} 00:00:00 +0000 1)# tags: [ tag1 ]
  142. 98637cd80f8c15 (Some R. Author {yesterday_date} 00:00:00 +0000 2)# tags: [ tag2 ]
  143. 3fcadac9d861d0 (Some R. Author {older_date} 00:00:00 +0000 3)# results: [ Failure ]
  144. 5f03bc04975c04 (Some R. Author {today_date} 00:00:00 +0000 4)
  145. 5f03bc04975c04 (Some R. Author {today_date} 00:00:00 +0000 5)crbug.com/1234 [ tag1 ] testname [ Failure ]
  146. 98637cd80f8c15 (Some R. Author {yesterday_date} 00:00:00 +0000 6)[ tag2 ] testname [ Failure ] # Comment
  147. 3fcadac9d861d0 (Some R. Author {older_date} 00:00:00 +0000 7)[ tag1 ] othertest [ Failure ]"""
  148. # pylint: enable=line-too-long
  149. blame_output = blame_output.format(today_date=today_str,
  150. yesterday_date=yesterday_str,
  151. older_date=older_str)
  152. self._output_mock.return_value = blame_output.encode('utf-8')
  153. expected_content = """\
  154. # tags: [ tag1 ]
  155. # tags: [ tag2 ]
  156. # results: [ Failure ]
  157. [ tag1 ] othertest [ Failure ]"""
  158. self.assertEqual(self.instance._GetNonRecentExpectationContent('', 1),
  159. expected_content)
  160. def testNegativeGracePeriod(self) -> None:
  161. """Tests that setting a negative grace period disables filtering."""
  162. today_date = datetime.date.today()
  163. yesterday_date = today_date - datetime.timedelta(days=1)
  164. older_date = today_date - datetime.timedelta(days=2)
  165. today_str = today_date.isoformat()
  166. yesterday_str = yesterday_date.isoformat()
  167. older_str = older_date.isoformat()
  168. # pylint: disable=line-too-long
  169. blame_output = """\
  170. 5f03bc04975c04 (Some R. Author {today_date} 00:00:00 +0000 1)# tags: [ tag1 ]
  171. 98637cd80f8c15 (Some R. Author {yesterday_date} 00:00:00 +0000 2)# tags: [ tag2 ]
  172. 3fcadac9d861d0 (Some R. Author {older_date} 00:00:00 +0000 3)# results: [ Failure ]
  173. 5f03bc04975c04 (Some R. Author {today_date} 00:00:00 +0000 4)
  174. 5f03bc04975c04 (Some R. Author {today_date} 00:00:00 +0000 5)crbug.com/1234 [ tag1 ] testname [ Failure ]
  175. 98637cd80f8c15 (Some R. Author {yesterday_date} 00:00:00 +0000 6)[ tag2 ] testname [ Failure ] # Comment
  176. 3fcadac9d861d0 (Some R. Author {older_date} 00:00:00 +0000 7)[ tag1 ] othertest [ Failure ]"""
  177. # pylint: enable=line-too-long
  178. blame_output = blame_output.format(today_date=today_str,
  179. yesterday_date=yesterday_str,
  180. older_date=older_str)
  181. self._output_mock.return_value = blame_output.encode('utf-8')
  182. expected_content = """\
  183. # tags: [ tag1 ]
  184. # tags: [ tag2 ]
  185. # results: [ Failure ]
  186. crbug.com/1234 [ tag1 ] testname [ Failure ]
  187. [ tag2 ] testname [ Failure ] # Comment
  188. [ tag1 ] othertest [ Failure ]"""
  189. self.assertEqual(self.instance._GetNonRecentExpectationContent('', -1),
  190. expected_content)
  191. class RemoveExpectationsFromFileUnittest(fake_filesystem_unittest.TestCase):
  192. def setUp(self) -> None:
  193. self.instance = uu.CreateGenericExpectations()
  194. self.header = self.instance._GetExpectationFileTagHeader(None)
  195. self.setUpPyfakefs()
  196. with tempfile.NamedTemporaryFile(delete=False) as f:
  197. self.filename = f.name
  198. def testExpectationRemoval(self) -> None:
  199. """Tests that expectations are properly removed from a file."""
  200. contents = self.header + """
  201. # This is a test comment
  202. crbug.com/1234 [ win ] foo/test [ Failure ]
  203. crbug.com/2345 [ win ] foo/test [ RetryOnFailure ]
  204. # Another comment
  205. [ linux ] bar/test [ RetryOnFailure ]
  206. [ win ] bar/test [ RetryOnFailure ]
  207. """
  208. stale_expectations = [
  209. data_types.Expectation('foo/test', ['win'], ['Failure'],
  210. 'crbug.com/1234'),
  211. data_types.Expectation('bar/test', ['linux'], ['RetryOnFailure'])
  212. ]
  213. expected_contents = self.header + """
  214. # This is a test comment
  215. crbug.com/2345 [ win ] foo/test [ RetryOnFailure ]
  216. # Another comment
  217. [ win ] bar/test [ RetryOnFailure ]
  218. """
  219. with open(self.filename, 'w') as f:
  220. f.write(contents)
  221. removed_urls = self.instance.RemoveExpectationsFromFile(
  222. stale_expectations, self.filename, expectations.RemovalType.STALE)
  223. self.assertEqual(removed_urls, set(['crbug.com/1234']))
  224. with open(self.filename) as f:
  225. self.assertEqual(f.read(), expected_contents)
  226. def testRemovalWithMultipleBugs(self) -> None:
  227. """Tests removal of expectations with multiple associated bugs."""
  228. contents = self.header + """
  229. # This is a test comment
  230. crbug.com/1234 crbug.com/3456 crbug.com/4567 [ win ] foo/test [ Failure ]
  231. crbug.com/2345 [ win ] foo/test [ RetryOnFailure ]
  232. # Another comment
  233. [ linux ] bar/test [ RetryOnFailure ]
  234. [ win ] bar/test [ RetryOnFailure ]
  235. """
  236. stale_expectations = [
  237. data_types.Expectation('foo/test', ['win'], ['Failure'],
  238. 'crbug.com/1234 crbug.com/3456 crbug.com/4567'),
  239. ]
  240. expected_contents = self.header + """
  241. # This is a test comment
  242. crbug.com/2345 [ win ] foo/test [ RetryOnFailure ]
  243. # Another comment
  244. [ linux ] bar/test [ RetryOnFailure ]
  245. [ win ] bar/test [ RetryOnFailure ]
  246. """
  247. with open(self.filename, 'w') as f:
  248. f.write(contents)
  249. removed_urls = self.instance.RemoveExpectationsFromFile(
  250. stale_expectations, self.filename, expectations.RemovalType.STALE)
  251. self.assertEqual(
  252. removed_urls,
  253. set(['crbug.com/1234', 'crbug.com/3456', 'crbug.com/4567']))
  254. with open(self.filename) as f:
  255. self.assertEqual(f.read(), expected_contents)
  256. def testNestedBlockComments(self) -> None:
  257. """Tests that nested disable block comments throw exceptions."""
  258. contents = self.header + """
  259. # finder:disable-general
  260. # finder:disable-general
  261. crbug.com/1234 [ win ] foo/test [ Failure ]
  262. # finder:enable-general
  263. # finder:enable-general
  264. """
  265. with open(self.filename, 'w') as f:
  266. f.write(contents)
  267. with self.assertRaises(RuntimeError):
  268. self.instance.RemoveExpectationsFromFile([], self.filename,
  269. expectations.RemovalType.STALE)
  270. contents = self.header + """
  271. # finder:disable-general
  272. # finder:disable-stale
  273. crbug.com/1234 [ win ] foo/test [ Failure ]
  274. # finder:enable-stale
  275. # finder:enable-genearl
  276. """
  277. with open(self.filename, 'w') as f:
  278. f.write(contents)
  279. with self.assertRaises(RuntimeError):
  280. self.instance.RemoveExpectationsFromFile([], self.filename,
  281. expectations.RemovalType.STALE)
  282. contents = self.header + """
  283. # finder:enable-general
  284. crbug.com/1234 [ win ] foo/test [ Failure ]
  285. """
  286. with open(self.filename, 'w') as f:
  287. f.write(contents)
  288. with self.assertRaises(RuntimeError):
  289. self.instance.RemoveExpectationsFromFile([], self.filename,
  290. expectations.RemovalType.STALE)
  291. def testGeneralBlockComments(self) -> None:
  292. """Tests that expectations in a disable block comment are not removed."""
  293. contents = self.header + """
  294. crbug.com/1234 [ win ] foo/test [ Failure ]
  295. # finder:disable-general
  296. crbug.com/2345 [ win ] foo/test [ Failure ]
  297. crbug.com/3456 [ win ] foo/test [ Failure ]
  298. # finder:enable-general
  299. crbug.com/4567 [ win ] foo/test [ Failure ]
  300. """
  301. stale_expectations = [
  302. data_types.Expectation('foo/test', ['win'], ['Failure'],
  303. 'crbug.com/1234'),
  304. data_types.Expectation('foo/test', ['win'], ['Failure'],
  305. 'crbug.com/2345'),
  306. data_types.Expectation('foo/test', ['win'], ['Failure'],
  307. 'crbug.com/3456'),
  308. data_types.Expectation('foo/test', ['win'], ['Failure'],
  309. 'crbug.com/4567'),
  310. ]
  311. expected_contents = self.header + """
  312. # finder:disable-general
  313. crbug.com/2345 [ win ] foo/test [ Failure ]
  314. crbug.com/3456 [ win ] foo/test [ Failure ]
  315. # finder:enable-general
  316. """
  317. for removal_type in (expectations.RemovalType.STALE,
  318. expectations.RemovalType.UNUSED):
  319. with open(self.filename, 'w') as f:
  320. f.write(contents)
  321. removed_urls = self.instance.RemoveExpectationsFromFile(
  322. stale_expectations, self.filename, removal_type)
  323. self.assertEqual(removed_urls, set(['crbug.com/1234', 'crbug.com/4567']))
  324. with open(self.filename) as f:
  325. self.assertEqual(f.read(), expected_contents)
  326. def testStaleBlockComments(self) -> None:
  327. """Tests that stale expectations in a stale disable block are not removed"""
  328. contents = self.header + """
  329. crbug.com/1234 [ win ] not_stale [ Failure ]
  330. crbug.com/1234 [ win ] before_block [ Failure ]
  331. # finder:disable-stale
  332. crbug.com/2345 [ win ] in_block [ Failure ]
  333. # finder:enable-stale
  334. crbug.com/3456 [ win ] after_block [ Failure ]
  335. """
  336. stale_expectations = [
  337. data_types.Expectation('before_block', ['win'], 'Failure',
  338. 'crbug.com/1234'),
  339. data_types.Expectation('in_block', ['win'], 'Failure',
  340. 'crbug.com/2345'),
  341. data_types.Expectation('after_block', ['win'], 'Failure',
  342. 'crbug.com/3456'),
  343. ]
  344. expected_contents = self.header + """
  345. crbug.com/1234 [ win ] not_stale [ Failure ]
  346. # finder:disable-stale
  347. crbug.com/2345 [ win ] in_block [ Failure ]
  348. # finder:enable-stale
  349. """
  350. with open(self.filename, 'w') as f:
  351. f.write(contents)
  352. removed_urls = self.instance.RemoveExpectationsFromFile(
  353. stale_expectations, self.filename, expectations.RemovalType.STALE)
  354. self.assertEqual(removed_urls, set(['crbug.com/1234', 'crbug.com/3456']))
  355. with open(self.filename) as f:
  356. self.assertEqual(f.read(), expected_contents)
  357. def testUnusedBlockComments(self) -> None:
  358. """Tests that stale expectations in unused disable blocks are not removed"""
  359. contents = self.header + """
  360. crbug.com/1234 [ win ] not_unused [ Failure ]
  361. crbug.com/1234 [ win ] before_block [ Failure ]
  362. # finder:disable-unused
  363. crbug.com/2345 [ win ] in_block [ Failure ]
  364. # finder:enable-unused
  365. crbug.com/3456 [ win ] after_block [ Failure ]
  366. """
  367. unused_expectations = [
  368. data_types.Expectation('before_block', ['win'], 'Failure',
  369. 'crbug.com/1234'),
  370. data_types.Expectation('in_block', ['win'], 'Failure',
  371. 'crbug.com/2345'),
  372. data_types.Expectation('after_block', ['win'], 'Failure',
  373. 'crbug.com/3456'),
  374. ]
  375. expected_contents = self.header + """
  376. crbug.com/1234 [ win ] not_unused [ Failure ]
  377. # finder:disable-unused
  378. crbug.com/2345 [ win ] in_block [ Failure ]
  379. # finder:enable-unused
  380. """
  381. with open(self.filename, 'w') as f:
  382. f.write(contents)
  383. removed_urls = self.instance.RemoveExpectationsFromFile(
  384. unused_expectations, self.filename, expectations.RemovalType.UNUSED)
  385. self.assertEqual(removed_urls, set(['crbug.com/1234', 'crbug.com/3456']))
  386. with open(self.filename) as f:
  387. self.assertEqual(f.read(), expected_contents)
  388. def testMismatchedBlockComments(self) -> None:
  389. """Tests that block comments for the wrong removal type do nothing."""
  390. contents = self.header + """
  391. crbug.com/1234 [ win ] do_not_remove [ Failure ]
  392. # finder:disable-stale
  393. crbug.com/2345 [ win ] disabled_stale [ Failure ]
  394. # finder:enable-stale
  395. # finder:disable-unused
  396. crbug.com/3456 [ win ] disabled_unused [ Failure ]
  397. # finder:enable-unused
  398. crbug.com/4567 [ win ] also_do_not_remove [ Failure ]
  399. """
  400. expectations_to_remove = [
  401. data_types.Expectation('disabled_stale', ['win'], 'Failure',
  402. 'crbug.com/2345'),
  403. data_types.Expectation('disabled_unused', ['win'], 'Failure',
  404. 'crbug.com/3456'),
  405. ]
  406. expected_contents = self.header + """
  407. crbug.com/1234 [ win ] do_not_remove [ Failure ]
  408. # finder:disable-stale
  409. crbug.com/2345 [ win ] disabled_stale [ Failure ]
  410. # finder:enable-stale
  411. # finder:disable-unused
  412. # finder:enable-unused
  413. crbug.com/4567 [ win ] also_do_not_remove [ Failure ]
  414. """
  415. with open(self.filename, 'w') as f:
  416. f.write(contents)
  417. removed_urls = self.instance.RemoveExpectationsFromFile(
  418. expectations_to_remove, self.filename, expectations.RemovalType.STALE)
  419. self.assertEqual(removed_urls, set(['crbug.com/3456']))
  420. with open(self.filename) as f:
  421. self.assertEqual(f.read(), expected_contents)
  422. expected_contents = self.header + """
  423. crbug.com/1234 [ win ] do_not_remove [ Failure ]
  424. # finder:disable-stale
  425. # finder:enable-stale
  426. # finder:disable-unused
  427. crbug.com/3456 [ win ] disabled_unused [ Failure ]
  428. # finder:enable-unused
  429. crbug.com/4567 [ win ] also_do_not_remove [ Failure ]
  430. """
  431. with open(self.filename, 'w') as f:
  432. f.write(contents)
  433. removed_urls = self.instance.RemoveExpectationsFromFile(
  434. expectations_to_remove, self.filename, expectations.RemovalType.UNUSED)
  435. self.assertEqual(removed_urls, set(['crbug.com/2345']))
  436. with open(self.filename) as f:
  437. self.assertEqual(f.read(), expected_contents)
  438. def testInlineGeneralComments(self) -> None:
  439. """Tests that expectations with inline disable comments are not removed."""
  440. contents = self.header + """
  441. crbug.com/1234 [ win ] foo/test [ Failure ]
  442. crbug.com/2345 [ win ] foo/test [ Failure ] # finder:disable-general
  443. crbug.com/3456 [ win ] foo/test [ Failure ]
  444. """
  445. stale_expectations = [
  446. data_types.Expectation('foo/test', ['win'], ['Failure'],
  447. 'crbug.com/1234'),
  448. data_types.Expectation('foo/test', ['win'], ['Failure'],
  449. 'crbug.com/2345'),
  450. data_types.Expectation('foo/test', ['win'], ['Failure'],
  451. 'crbug.com/3456'),
  452. ]
  453. expected_contents = self.header + """
  454. crbug.com/2345 [ win ] foo/test [ Failure ] # finder:disable-general
  455. """
  456. for removal_type in (expectations.RemovalType.STALE,
  457. expectations.RemovalType.UNUSED):
  458. with open(self.filename, 'w') as f:
  459. f.write(contents)
  460. removed_urls = self.instance.RemoveExpectationsFromFile(
  461. stale_expectations, self.filename, removal_type)
  462. self.assertEqual(removed_urls, set(['crbug.com/1234', 'crbug.com/3456']))
  463. with open(self.filename) as f:
  464. self.assertEqual(f.read(), expected_contents)
  465. def testInlineStaleComments(self) -> None:
  466. """Tests that expectations with inline stale disable comments not removed"""
  467. contents = self.header + """
  468. crbug.com/1234 [ win ] not_disabled [ Failure ]
  469. crbug.com/2345 [ win ] stale_disabled [ Failure ] # finder:disable-stale
  470. crbug.com/3456 [ win ] unused_disabled [ Failure ] # finder:disable-unused
  471. """
  472. stale_expectations = [
  473. data_types.Expectation('not_disabled', ['win'], 'Failure',
  474. 'crbug.com/1234'),
  475. data_types.Expectation('stale_disabled', ['win'], 'Failure',
  476. 'crbug.com/2345'),
  477. data_types.Expectation('unused_disabled', ['win'], 'Failure',
  478. 'crbug.com/3456')
  479. ]
  480. expected_contents = self.header + """
  481. crbug.com/2345 [ win ] stale_disabled [ Failure ] # finder:disable-stale
  482. """
  483. with open(self.filename, 'w') as f:
  484. f.write(contents)
  485. removed_urls = self.instance.RemoveExpectationsFromFile(
  486. stale_expectations, self.filename, expectations.RemovalType.STALE)
  487. self.assertEqual(removed_urls, set(['crbug.com/1234', 'crbug.com/3456']))
  488. with open(self.filename) as f:
  489. self.assertEqual(f.read(), expected_contents)
  490. def testInlineUnusedComments(self) -> None:
  491. """Tests that expectations with inline unused comments not removed"""
  492. contents = self.header + """
  493. crbug.com/1234 [ win ] not_disabled [ Failure ]
  494. crbug.com/2345 [ win ] stale_disabled [ Failure ] # finder:disable-stale
  495. crbug.com/3456 [ win ] unused_disabled [ Failure ] # finder:disable-unused
  496. """
  497. stale_expectations = [
  498. data_types.Expectation('not_disabled', ['win'], 'Failure',
  499. 'crbug.com/1234'),
  500. data_types.Expectation('stale_disabled', ['win'], 'Failure',
  501. 'crbug.com/2345'),
  502. data_types.Expectation('unused_disabled', ['win'], 'Failure',
  503. 'crbug.com/3456')
  504. ]
  505. expected_contents = self.header + """
  506. crbug.com/3456 [ win ] unused_disabled [ Failure ] # finder:disable-unused
  507. """
  508. with open(self.filename, 'w') as f:
  509. f.write(contents)
  510. removed_urls = self.instance.RemoveExpectationsFromFile(
  511. stale_expectations, self.filename, expectations.RemovalType.UNUSED)
  512. self.assertEqual(removed_urls, set(['crbug.com/1234', 'crbug.com/2345']))
  513. with open(self.filename) as f:
  514. self.assertEqual(f.read(), expected_contents)
  515. def testGetDisableReasonFromComment(self):
  516. """Tests that the disable reason can be pulled from a line."""
  517. self.assertEqual(
  518. expectations._GetDisableReasonFromComment(
  519. '# finder:disable-general foo'), 'foo')
  520. self.assertEqual(
  521. expectations._GetDisableReasonFromComment(
  522. 'crbug.com/1234 [ win ] bar/test [ Failure ] '
  523. '# finder:disable-general foo'), 'foo')
  524. class GetExpectationLineUnittest(unittest.TestCase):
  525. def setUp(self) -> None:
  526. self.instance = uu.CreateGenericExpectations()
  527. def testNoMatchingExpectation(self) -> None:
  528. """Tests that the case of no matching expectation is handled."""
  529. expectation = data_types.Expectation('foo', ['win'], 'Failure')
  530. line, line_number = self.instance._GetExpectationLine(
  531. expectation, FAKE_EXPECTATION_FILE_CONTENTS, 'expectation_file')
  532. self.assertIsNone(line)
  533. self.assertIsNone(line_number)
  534. def testMatchingExpectation(self) -> None:
  535. """Tests that matching expectations are found."""
  536. expectation = data_types.Expectation('foo/test', ['win'], 'Failure',
  537. 'crbug.com/1234')
  538. line, line_number = self.instance._GetExpectationLine(
  539. expectation, FAKE_EXPECTATION_FILE_CONTENTS, 'expectation_file')
  540. self.assertEqual(line, 'crbug.com/1234 [ win ] foo/test [ Failure ]')
  541. self.assertEqual(line_number, 3)
  542. class ModifySemiStaleExpectationsUnittest(fake_filesystem_unittest.TestCase):
  543. def setUp(self) -> None:
  544. self.setUpPyfakefs()
  545. self.instance = uu.CreateGenericExpectations()
  546. self._input_patcher = mock.patch.object(expectations,
  547. '_WaitForUserInputOnModification')
  548. self._input_mock = self._input_patcher.start()
  549. self.addCleanup(self._input_patcher.stop)
  550. with tempfile.NamedTemporaryFile(delete=False, mode='w') as f:
  551. f.write(FAKE_EXPECTATION_FILE_CONTENTS)
  552. self.filename = f.name
  553. with tempfile.NamedTemporaryFile(delete=False, mode='w') as f:
  554. f.write(SECONDARY_FAKE_EXPECTATION_FILE_CONTENTS)
  555. self.secondary_filename = f.name
  556. def testEmptyExpectationMap(self) -> None:
  557. """Tests that an empty expectation map results in a no-op."""
  558. modified_urls = self.instance.ModifySemiStaleExpectations(
  559. data_types.TestExpectationMap())
  560. self.assertEqual(modified_urls, set())
  561. self._input_mock.assert_not_called()
  562. with open(self.filename) as f:
  563. self.assertEqual(f.read(), FAKE_EXPECTATION_FILE_CONTENTS)
  564. def testRemoveExpectation(self) -> None:
  565. """Tests that specifying to remove an expectation does so."""
  566. self._input_mock.return_value = 'r'
  567. # yapf: disable
  568. test_expectation_map = data_types.TestExpectationMap({
  569. self.filename:
  570. data_types.ExpectationBuilderMap({
  571. data_types.Expectation(
  572. 'foo/test', ['win'], 'Failure', 'crbug.com/1234'):
  573. data_types.BuilderStepMap(),
  574. }),
  575. self.secondary_filename:
  576. data_types.ExpectationBuilderMap({
  577. data_types.Expectation(
  578. 'foo/test', ['mac'], 'Failure', 'crbug.com/4567'):
  579. data_types.BuilderStepMap(),
  580. }),
  581. })
  582. # yapf: enable
  583. modified_urls = self.instance.ModifySemiStaleExpectations(
  584. test_expectation_map)
  585. self.assertEqual(modified_urls, set(['crbug.com/1234', 'crbug.com/4567']))
  586. expected_file_contents = """\
  587. # tags: [ win linux ]
  588. # results: [ Failure RetryOnFailure Skip Pass ]
  589. crbug.com/5678 crbug.com/6789 [ win ] foo/another/test [ RetryOnFailure ]
  590. [ linux ] foo/test [ Failure ]
  591. crbug.com/2345 [ linux ] bar/* [ RetryOnFailure ]
  592. crbug.com/3456 [ linux ] some/bad/test [ Skip ]
  593. crbug.com/4567 [ linux ] some/good/test [ Pass ]
  594. """
  595. with open(self.filename) as f:
  596. self.assertEqual(f.read(), expected_file_contents)
  597. expected_file_contents = """\
  598. # tags: [ mac ]
  599. # results: [ Failure ]
  600. """
  601. with open(self.secondary_filename) as f:
  602. self.assertEqual(f.read(), expected_file_contents)
  603. def testModifyExpectation(self) -> None:
  604. """Tests that specifying to modify an expectation does not remove it."""
  605. self._input_mock.return_value = 'm'
  606. # yapf: disable
  607. test_expectation_map = data_types.TestExpectationMap({
  608. self.filename:
  609. data_types.ExpectationBuilderMap({
  610. data_types.Expectation(
  611. 'foo/test', ['win'], 'Failure', 'crbug.com/1234'):
  612. data_types.BuilderStepMap(),
  613. }),
  614. self.secondary_filename:
  615. data_types.ExpectationBuilderMap({
  616. data_types.Expectation(
  617. 'foo/test', ['mac'], 'Failure', 'crbug.com/4567',
  618. ): data_types.BuilderStepMap()
  619. }),
  620. })
  621. # yapf: enable
  622. modified_urls = self.instance.ModifySemiStaleExpectations(
  623. test_expectation_map)
  624. self.assertEqual(modified_urls, set(['crbug.com/1234', 'crbug.com/4567']))
  625. with open(self.filename) as f:
  626. self.assertEqual(f.read(), FAKE_EXPECTATION_FILE_CONTENTS)
  627. with open(self.secondary_filename) as f:
  628. self.assertEqual(f.read(), SECONDARY_FAKE_EXPECTATION_FILE_CONTENTS)
  629. def testModifyExpectationMultipleBugs(self) -> None:
  630. """Tests that modifying an expectation with multiple bugs works properly."""
  631. self._input_mock.return_value = 'm'
  632. # yapf: disable
  633. test_expectation_map = data_types.TestExpectationMap({
  634. self.filename:
  635. data_types.ExpectationBuilderMap({
  636. data_types.Expectation(
  637. 'foo/another/test', ['win'], 'RetryOnFailure',
  638. 'crbug.com/5678 crbug.com/6789'):
  639. data_types.BuilderStepMap(),
  640. }),
  641. })
  642. # yapf: enable
  643. modified_urls = self.instance.ModifySemiStaleExpectations(
  644. test_expectation_map)
  645. self.assertEqual(modified_urls, set(['crbug.com/5678', 'crbug.com/6789']))
  646. with open(self.filename) as f:
  647. self.assertEqual(f.read(), FAKE_EXPECTATION_FILE_CONTENTS)
  648. with open(self.secondary_filename) as f:
  649. self.assertEqual(f.read(), SECONDARY_FAKE_EXPECTATION_FILE_CONTENTS)
  650. def testIgnoreExpectation(self) -> None:
  651. """Tests that specifying to ignore an expectation does nothing."""
  652. self._input_mock.return_value = 'i'
  653. # yapf: disable
  654. test_expectation_map = data_types.TestExpectationMap({
  655. self.filename:
  656. data_types.ExpectationBuilderMap({
  657. data_types.Expectation(
  658. 'foo/test', ['win'], 'Failure', 'crbug.com/1234'):
  659. data_types.BuilderStepMap(),
  660. }),
  661. self.secondary_filename:
  662. data_types.ExpectationBuilderMap({
  663. data_types.Expectation(
  664. 'foo/test', ['mac'], 'Failure', 'crbug.com/4567',
  665. ): data_types.BuilderStepMap()
  666. }),
  667. })
  668. # yapf: enable
  669. modified_urls = self.instance.ModifySemiStaleExpectations(
  670. test_expectation_map)
  671. self.assertEqual(modified_urls, set())
  672. with open(self.filename) as f:
  673. self.assertEqual(f.read(), FAKE_EXPECTATION_FILE_CONTENTS)
  674. with open(self.secondary_filename) as f:
  675. self.assertEqual(f.read(), SECONDARY_FAKE_EXPECTATION_FILE_CONTENTS)
  676. def testParserErrorCorrection(self) -> None:
  677. """Tests that parser errors are caught and users can fix them."""
  678. def TypoSideEffect() -> str:
  679. with open(self.filename, 'w') as outfile:
  680. outfile.write(FAKE_EXPECTATION_FILE_CONTENTS_WITH_TYPO)
  681. return 'm'
  682. def CorrectionSideEffect() -> None:
  683. with open(self.filename, 'w') as outfile:
  684. outfile.write(FAKE_EXPECTATION_FILE_CONTENTS)
  685. self._input_mock.side_effect = TypoSideEffect
  686. with mock.patch.object(expectations,
  687. '_WaitForAnyUserInput') as any_input_mock:
  688. any_input_mock.side_effect = CorrectionSideEffect
  689. # yapf: disable
  690. test_expectation_map = data_types.TestExpectationMap({
  691. self.filename:
  692. data_types.ExpectationBuilderMap({
  693. data_types.Expectation(
  694. 'foo/test', ['win'], 'Failure', 'crbug.com/1234'):
  695. data_types.BuilderStepMap(),
  696. }),
  697. })
  698. # yapf: enable
  699. self.instance.ModifySemiStaleExpectations(test_expectation_map)
  700. any_input_mock.assert_called_once()
  701. with open(self.filename) as infile:
  702. self.assertEqual(infile.read(), FAKE_EXPECTATION_FILE_CONTENTS)
  703. class FindOrphanedBugsUnittest(fake_filesystem_unittest.TestCase):
  704. def CreateFile(self, *args, **kwargs) -> None:
  705. # TODO(crbug.com/1156806): Remove this and just use fs.create_file() when
  706. # Catapult is updated to a newer version of pyfakefs that is compatible with
  707. # Chromium's version.
  708. if hasattr(self.fs, 'create_file'):
  709. self.fs.create_file(*args, **kwargs)
  710. else:
  711. self.fs.CreateFile(*args, **kwargs)
  712. def setUp(self) -> None:
  713. expectations_dir = os.path.join(os.path.dirname(__file__), 'expectations')
  714. self.setUpPyfakefs()
  715. self.instance = expectations.Expectations()
  716. self.filepath_patcher = mock.patch.object(
  717. self.instance,
  718. 'GetExpectationFilepaths',
  719. return_value=[os.path.join(expectations_dir, 'real_expectations.txt')])
  720. self.filepath_mock = self.filepath_patcher.start()
  721. self.addCleanup(self.filepath_patcher.stop)
  722. real_contents = 'crbug.com/1\ncrbug.com/2'
  723. skipped_contents = 'crbug.com/4'
  724. self.CreateFile(os.path.join(expectations_dir, 'real_expectations.txt'),
  725. contents=real_contents)
  726. self.CreateFile(os.path.join(expectations_dir, 'fake.txt'),
  727. contents=skipped_contents)
  728. def testNoOrphanedBugs(self) -> None:
  729. bugs = ['crbug.com/1', 'crbug.com/2']
  730. self.assertEqual(self.instance.FindOrphanedBugs(bugs), set())
  731. def testOrphanedBugs(self) -> None:
  732. bugs = ['crbug.com/1', 'crbug.com/3', 'crbug.com/4']
  733. self.assertEqual(self.instance.FindOrphanedBugs(bugs),
  734. set(['crbug.com/3', 'crbug.com/4']))
  735. if __name__ == '__main__':
  736. unittest.main(verbosity=2)