test_filtering_unittests.py 11 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299
  1. #!/usr/bin/env vpython3
  2. # Copyright 2021 The Chromium Authors. All rights reserved.
  3. # Use of this source code is governed by a BSD-style license that can be
  4. # found in the LICENSE file.
  5. import argparse
  6. import os
  7. from pyfakefs import fake_filesystem_unittest
  8. import tempfile
  9. import unittest
  10. import test_filtering
  11. from test_filtering import _TestFilter
  12. from test_filtering import _TestFiltersGroup
  13. from test_filtering import _SetOfTestFiltersGroups
  14. class FilterTests(fake_filesystem_unittest.TestCase):
  15. def test_exact_match(self):
  16. t = _TestFilter('foo')
  17. self.assertTrue(t.is_match('foo'))
  18. self.assertFalse(t.is_match('foobar'))
  19. self.assertFalse(t.is_match('foo/bar'))
  20. self.assertFalse(t.is_match('fo'))
  21. self.assertFalse(t.is_exclusion_filter())
  22. def test_prefix_match(self):
  23. t = _TestFilter('foo*')
  24. self.assertTrue(t.is_match('foo'))
  25. self.assertTrue(t.is_match('foobar'))
  26. self.assertTrue(t.is_match('foo/bar'))
  27. self.assertFalse(t.is_match('fo'))
  28. self.assertFalse(t.is_exclusion_filter())
  29. def test_exclusion_match(self):
  30. t = _TestFilter('-foo')
  31. self.assertTrue(t.is_match('foo'))
  32. self.assertFalse(t.is_match('foobar'))
  33. self.assertFalse(t.is_match('foo/bar'))
  34. self.assertFalse(t.is_match('fo'))
  35. self.assertTrue(t.is_exclusion_filter())
  36. def test_exclusion_prefix_match(self):
  37. t = _TestFilter('-foo*')
  38. self.assertTrue(t.is_match('foo'))
  39. self.assertTrue(t.is_match('foobar'))
  40. self.assertTrue(t.is_match('foo/bar'))
  41. self.assertFalse(t.is_match('fo'))
  42. self.assertTrue(t.is_exclusion_filter())
  43. def test_error_conditions(self):
  44. # '*' is only supported at the end
  45. with self.assertRaises(ValueError):
  46. _TestFilter('*.bar')
  47. def _create_group_from_pseudo_file(file_contents):
  48. # pylint: disable=unexpected-keyword-arg
  49. with tempfile.NamedTemporaryFile(delete=False, mode='w',
  50. encoding='utf-8') as f:
  51. filepath = f.name
  52. f.write(file_contents)
  53. try:
  54. return _TestFiltersGroup.from_filter_file(filepath)
  55. finally:
  56. os.remove(filepath)
  57. class FiltersGroupTests(fake_filesystem_unittest.TestCase):
  58. def test_single_positive_filter(self):
  59. g = _TestFiltersGroup.from_string('foo*')
  60. self.assertTrue(g.is_test_name_included('foo'))
  61. self.assertTrue(g.is_test_name_included('foobar'))
  62. self.assertFalse(g.is_test_name_included('baz'))
  63. self.assertFalse(g.is_test_name_included('fo'))
  64. def test_single_negative_filter(self):
  65. g = _TestFiltersGroup.from_string('-foo*')
  66. self.assertFalse(g.is_test_name_included('foo'))
  67. self.assertFalse(g.is_test_name_included('foobar'))
  68. self.assertTrue(g.is_test_name_included('baz'))
  69. self.assertTrue(g.is_test_name_included('fo'))
  70. def test_specificity_ordering(self):
  71. # From test_executable_api.md#filtering-which-tests-to-run:
  72. #
  73. # If multiple filters in a flag match a given test name, the longest
  74. # match takes priority (longest match wins). I.e.,. if you had
  75. # --isolated-script-test-filter='a*::-ab*', then ace.html would run
  76. # but abd.html would not. The order of the filters should not
  77. # matter.
  78. g1 = _TestFiltersGroup.from_string('a*::-ab*') # order #1
  79. g2 = _TestFiltersGroup.from_string('-ab*::a*') # order #2
  80. self.assertTrue(g1.is_test_name_included('ace'))
  81. self.assertTrue(g2.is_test_name_included('ace'))
  82. self.assertFalse(g1.is_test_name_included('abd'))
  83. self.assertFalse(g2.is_test_name_included('abd'))
  84. def test_specificity_conflicts(self):
  85. # Docs give this specific example: It is an error to have multiple
  86. # expressions of the same length that conflict (e.g., a*::-a*).
  87. with self.assertRaises(ValueError):
  88. _TestFiltersGroup.from_string('a*::-a*')
  89. # Other similar conflict:
  90. with self.assertRaises(ValueError):
  91. _TestFiltersGroup.from_string('a::-a')
  92. # It doesn't really make sense to support `foo.bar` and `foo.bar*` and
  93. # have the latter take precedence over the former.
  94. with self.assertRaises(ValueError):
  95. _TestFiltersGroup.from_string('foo.bar::foo.bar*')
  96. # In the same spirit, identical duplicates are also treated as
  97. # conflicts.
  98. with self.assertRaises(ValueError):
  99. _TestFiltersGroup.from_string('foo.bar::foo.bar')
  100. # Ok - no conflicts:
  101. _TestFiltersGroup.from_string('a*::-b*') # Different filter text
  102. def test_simple_list(self):
  103. # 'simple test list' format from bit.ly/chromium-test-list-format
  104. # (aka go/test-list-format)
  105. file_content = """
  106. # Comment
  107. aaa
  108. bbb # End-of-line comment
  109. Bug(intentional) ccc [ Crash ] # Comment
  110. crbug.com/12345 [ Debug] ddd
  111. skbug.com/12345 [ Debug] eee
  112. webkit.org/12345 [ Debug] fff
  113. ggg*
  114. -ggghhh
  115. """.strip()
  116. g = _create_group_from_pseudo_file(file_content)
  117. self.assertTrue(g.is_test_name_included('aaa'))
  118. self.assertTrue(g.is_test_name_included('bbb'))
  119. self.assertTrue(g.is_test_name_included('ccc'))
  120. self.assertTrue(g.is_test_name_included('ddd'))
  121. self.assertFalse(g.is_test_name_included('aa'))
  122. self.assertFalse(g.is_test_name_included('aaax'))
  123. self.assertTrue(g.is_test_name_included('ggg'))
  124. self.assertTrue(g.is_test_name_included('gggg'))
  125. self.assertFalse(g.is_test_name_included('gg'))
  126. self.assertFalse(g.is_test_name_included('ggghhh'))
  127. self.assertFalse(g.is_test_name_included('zzz'))
  128. def test_tagged_list(self):
  129. # tagged list format from bit.ly/chromium-test-list-format
  130. # (aka go/test-list-format)
  131. file_content = """
  132. # Comment
  133. abc
  134. foo* # End-of-line comment
  135. -foobar*
  136. """.strip()
  137. g = _create_group_from_pseudo_file(file_content)
  138. self.assertTrue(g.is_test_name_included('abc'))
  139. self.assertFalse(g.is_test_name_included('abcd'))
  140. self.assertTrue(g.is_test_name_included('foo'))
  141. self.assertTrue(g.is_test_name_included('food'))
  142. self.assertFalse(g.is_test_name_included('foobar'))
  143. self.assertFalse(g.is_test_name_included('foobarbaz'))
  144. class SetOfFilterGroupsTests(fake_filesystem_unittest.TestCase):
  145. @classmethod
  146. def setUpClass(cls):
  147. # `_filter1`, `_filter2`, and `_tests` are based on the setup described
  148. # in test_executable_api.md#examples
  149. cls._filter1 = _TestFiltersGroup.from_string(
  150. 'Foo.Bar.*::-Foo.Bar.bar3')
  151. cls._filter2 = _TestFiltersGroup.from_string('Foo.Bar.bar2')
  152. cls._tests = [
  153. 'Foo.Bar.bar1',
  154. 'Foo.Bar.bar2',
  155. 'Foo.Bar.bar3',
  156. 'Foo.Baz.baz', # TODO: Fix typo in test_executable_api.md
  157. 'Foo.Quux.quux'
  158. ]
  159. def test_basics(self):
  160. # This test corresponds to
  161. # test_executable_api.md#filtering-tests-on-the-command-line
  162. # and
  163. # test_executable_api.md#using-a-filter-file
  164. s = _SetOfTestFiltersGroups([self._filter1])
  165. self.assertEqual(['Foo.Bar.bar1', 'Foo.Bar.bar2'],
  166. s.filter_test_names(self._tests))
  167. def test_combining_multiple_filters1(self):
  168. # This test corresponds to the first test under
  169. # test_executable_api.md#combining-multiple-filters
  170. s = _SetOfTestFiltersGroups([
  171. _TestFiltersGroup.from_string('Foo.Bar.*'),
  172. _TestFiltersGroup.from_string('Foo.Bar.bar2')
  173. ])
  174. self.assertEqual(['Foo.Bar.bar2'], s.filter_test_names(self._tests))
  175. def test_combining_multiple_filters2(self):
  176. # This test corresponds to the second test under
  177. # test_executable_api.md#combining-multiple-filters
  178. # TODO(lukasza@chromium.org): Figure out if the 3rd test example from
  179. # the docs has correct inputs+outputs (or if there are some typos).
  180. s = _SetOfTestFiltersGroups([
  181. _TestFiltersGroup.from_string('Foo.Bar.*'),
  182. _TestFiltersGroup.from_string('Foo.Baz.baz')
  183. ])
  184. self.assertEqual([], s.filter_test_names(self._tests))
  185. class PublicApiTests(fake_filesystem_unittest.TestCase):
  186. def test_filter_cmdline_arg(self):
  187. parser = argparse.ArgumentParser()
  188. test_filtering.add_cmdline_args(parser)
  189. args = parser.parse_args(args=[
  190. '--isolated-script-test-filter=-barbaz',
  191. '--isolated-script-test-filter=foo*::bar*'
  192. ])
  193. self.assertEqual(
  194. ['foo1', 'foo2', 'bar1', 'bar2'],
  195. test_filtering.filter_tests(
  196. args, {}, ['foo1', 'foo2', 'bar1', 'bar2', 'barbaz', 'zzz']))
  197. def test_filter_file_cmdline_arg(self):
  198. # pylint: disable=unexpected-keyword-arg
  199. f = tempfile.NamedTemporaryFile(delete=False,
  200. mode='w',
  201. encoding='utf-8')
  202. try:
  203. filepath = f.name
  204. f.write('foo*')
  205. f.close()
  206. parser = argparse.ArgumentParser()
  207. test_filtering.add_cmdline_args(parser)
  208. args = parser.parse_args(args=[
  209. '--isolated-script-test-filter-file={0:s}'.format(filepath)
  210. ])
  211. self.assertEqual(['foo1', 'foo2'],
  212. test_filtering.filter_tests(
  213. args, {}, ['foo1', 'foo2', 'bar1', 'bar2']))
  214. finally:
  215. os.remove(filepath)
  216. def _shard_tests(input_test_list_string, input_env):
  217. input_test_list = input_test_list_string.split(',')
  218. output_test_list = test_filtering._shard_tests(input_test_list, input_env)
  219. return ','.join(output_test_list)
  220. class ShardingTest(unittest.TestCase):
  221. def test_empty_environment(self):
  222. self.assertEqual('a,b,c', _shard_tests('a,b,c', {}))
  223. def test_basic_sharding(self):
  224. self.assertEqual(
  225. 'a,c,e',
  226. _shard_tests('a,b,c,d,e', {
  227. 'GTEST_SHARD_INDEX': '0',
  228. 'GTEST_TOTAL_SHARDS': '2'
  229. }))
  230. self.assertEqual(
  231. 'b,d',
  232. _shard_tests('a,b,c,d,e', {
  233. 'GTEST_SHARD_INDEX': '1',
  234. 'GTEST_TOTAL_SHARDS': '2'
  235. }))
  236. def test_error_conditions(self):
  237. # shard index > total shards
  238. with self.assertRaises(Exception):
  239. _shard_tests('', {
  240. 'GTEST_SHARD_INDEX': '2',
  241. 'GTEST_TOTAL_SHARDS': '2'
  242. })
  243. # non-integer shard index
  244. with self.assertRaises(Exception):
  245. _shard_tests('', {
  246. 'GTEST_SHARD_INDEX': 'a',
  247. 'GTEST_TOTAL_SHARDS': '2'
  248. })
  249. # non-integer total shards
  250. with self.assertRaises(Exception):
  251. _shard_tests('', {
  252. 'GTEST_SHARD_INDEX': '0',
  253. 'GTEST_TOTAL_SHARDS': 'b'
  254. })
  255. if __name__ == '__main__':
  256. unittest.main()