123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299 |
- #!/usr/bin/env vpython3
- # Copyright 2021 The Chromium Authors. All rights reserved.
- # Use of this source code is governed by a BSD-style license that can be
- # found in the LICENSE file.
- import argparse
- import os
- from pyfakefs import fake_filesystem_unittest
- import tempfile
- import unittest
- import test_filtering
- from test_filtering import _TestFilter
- from test_filtering import _TestFiltersGroup
- from test_filtering import _SetOfTestFiltersGroups
- class FilterTests(fake_filesystem_unittest.TestCase):
- def test_exact_match(self):
- t = _TestFilter('foo')
- self.assertTrue(t.is_match('foo'))
- self.assertFalse(t.is_match('foobar'))
- self.assertFalse(t.is_match('foo/bar'))
- self.assertFalse(t.is_match('fo'))
- self.assertFalse(t.is_exclusion_filter())
- def test_prefix_match(self):
- t = _TestFilter('foo*')
- self.assertTrue(t.is_match('foo'))
- self.assertTrue(t.is_match('foobar'))
- self.assertTrue(t.is_match('foo/bar'))
- self.assertFalse(t.is_match('fo'))
- self.assertFalse(t.is_exclusion_filter())
- def test_exclusion_match(self):
- t = _TestFilter('-foo')
- self.assertTrue(t.is_match('foo'))
- self.assertFalse(t.is_match('foobar'))
- self.assertFalse(t.is_match('foo/bar'))
- self.assertFalse(t.is_match('fo'))
- self.assertTrue(t.is_exclusion_filter())
- def test_exclusion_prefix_match(self):
- t = _TestFilter('-foo*')
- self.assertTrue(t.is_match('foo'))
- self.assertTrue(t.is_match('foobar'))
- self.assertTrue(t.is_match('foo/bar'))
- self.assertFalse(t.is_match('fo'))
- self.assertTrue(t.is_exclusion_filter())
- def test_error_conditions(self):
- # '*' is only supported at the end
- with self.assertRaises(ValueError):
- _TestFilter('*.bar')
- def _create_group_from_pseudo_file(file_contents):
- # pylint: disable=unexpected-keyword-arg
- with tempfile.NamedTemporaryFile(delete=False, mode='w',
- encoding='utf-8') as f:
- filepath = f.name
- f.write(file_contents)
- try:
- return _TestFiltersGroup.from_filter_file(filepath)
- finally:
- os.remove(filepath)
- class FiltersGroupTests(fake_filesystem_unittest.TestCase):
- def test_single_positive_filter(self):
- g = _TestFiltersGroup.from_string('foo*')
- self.assertTrue(g.is_test_name_included('foo'))
- self.assertTrue(g.is_test_name_included('foobar'))
- self.assertFalse(g.is_test_name_included('baz'))
- self.assertFalse(g.is_test_name_included('fo'))
- def test_single_negative_filter(self):
- g = _TestFiltersGroup.from_string('-foo*')
- self.assertFalse(g.is_test_name_included('foo'))
- self.assertFalse(g.is_test_name_included('foobar'))
- self.assertTrue(g.is_test_name_included('baz'))
- self.assertTrue(g.is_test_name_included('fo'))
- def test_specificity_ordering(self):
- # From test_executable_api.md#filtering-which-tests-to-run:
- #
- # If multiple filters in a flag match a given test name, the longest
- # match takes priority (longest match wins). I.e.,. if you had
- # --isolated-script-test-filter='a*::-ab*', then ace.html would run
- # but abd.html would not. The order of the filters should not
- # matter.
- g1 = _TestFiltersGroup.from_string('a*::-ab*') # order #1
- g2 = _TestFiltersGroup.from_string('-ab*::a*') # order #2
- self.assertTrue(g1.is_test_name_included('ace'))
- self.assertTrue(g2.is_test_name_included('ace'))
- self.assertFalse(g1.is_test_name_included('abd'))
- self.assertFalse(g2.is_test_name_included('abd'))
- def test_specificity_conflicts(self):
- # Docs give this specific example: It is an error to have multiple
- # expressions of the same length that conflict (e.g., a*::-a*).
- with self.assertRaises(ValueError):
- _TestFiltersGroup.from_string('a*::-a*')
- # Other similar conflict:
- with self.assertRaises(ValueError):
- _TestFiltersGroup.from_string('a::-a')
- # It doesn't really make sense to support `foo.bar` and `foo.bar*` and
- # have the latter take precedence over the former.
- with self.assertRaises(ValueError):
- _TestFiltersGroup.from_string('foo.bar::foo.bar*')
- # In the same spirit, identical duplicates are also treated as
- # conflicts.
- with self.assertRaises(ValueError):
- _TestFiltersGroup.from_string('foo.bar::foo.bar')
- # Ok - no conflicts:
- _TestFiltersGroup.from_string('a*::-b*') # Different filter text
- def test_simple_list(self):
- # 'simple test list' format from bit.ly/chromium-test-list-format
- # (aka go/test-list-format)
- file_content = """
- # Comment
- aaa
- bbb # End-of-line comment
- Bug(intentional) ccc [ Crash ] # Comment
- crbug.com/12345 [ Debug] ddd
- skbug.com/12345 [ Debug] eee
- webkit.org/12345 [ Debug] fff
- ggg*
- -ggghhh
- """.strip()
- g = _create_group_from_pseudo_file(file_content)
- self.assertTrue(g.is_test_name_included('aaa'))
- self.assertTrue(g.is_test_name_included('bbb'))
- self.assertTrue(g.is_test_name_included('ccc'))
- self.assertTrue(g.is_test_name_included('ddd'))
- self.assertFalse(g.is_test_name_included('aa'))
- self.assertFalse(g.is_test_name_included('aaax'))
- self.assertTrue(g.is_test_name_included('ggg'))
- self.assertTrue(g.is_test_name_included('gggg'))
- self.assertFalse(g.is_test_name_included('gg'))
- self.assertFalse(g.is_test_name_included('ggghhh'))
- self.assertFalse(g.is_test_name_included('zzz'))
- def test_tagged_list(self):
- # tagged list format from bit.ly/chromium-test-list-format
- # (aka go/test-list-format)
- file_content = """
- # Comment
- abc
- foo* # End-of-line comment
- -foobar*
- """.strip()
- g = _create_group_from_pseudo_file(file_content)
- self.assertTrue(g.is_test_name_included('abc'))
- self.assertFalse(g.is_test_name_included('abcd'))
- self.assertTrue(g.is_test_name_included('foo'))
- self.assertTrue(g.is_test_name_included('food'))
- self.assertFalse(g.is_test_name_included('foobar'))
- self.assertFalse(g.is_test_name_included('foobarbaz'))
- class SetOfFilterGroupsTests(fake_filesystem_unittest.TestCase):
- @classmethod
- def setUpClass(cls):
- # `_filter1`, `_filter2`, and `_tests` are based on the setup described
- # in test_executable_api.md#examples
- cls._filter1 = _TestFiltersGroup.from_string(
- 'Foo.Bar.*::-Foo.Bar.bar3')
- cls._filter2 = _TestFiltersGroup.from_string('Foo.Bar.bar2')
- cls._tests = [
- 'Foo.Bar.bar1',
- 'Foo.Bar.bar2',
- 'Foo.Bar.bar3',
- 'Foo.Baz.baz', # TODO: Fix typo in test_executable_api.md
- 'Foo.Quux.quux'
- ]
- def test_basics(self):
- # This test corresponds to
- # test_executable_api.md#filtering-tests-on-the-command-line
- # and
- # test_executable_api.md#using-a-filter-file
- s = _SetOfTestFiltersGroups([self._filter1])
- self.assertEqual(['Foo.Bar.bar1', 'Foo.Bar.bar2'],
- s.filter_test_names(self._tests))
- def test_combining_multiple_filters1(self):
- # This test corresponds to the first test under
- # test_executable_api.md#combining-multiple-filters
- s = _SetOfTestFiltersGroups([
- _TestFiltersGroup.from_string('Foo.Bar.*'),
- _TestFiltersGroup.from_string('Foo.Bar.bar2')
- ])
- self.assertEqual(['Foo.Bar.bar2'], s.filter_test_names(self._tests))
- def test_combining_multiple_filters2(self):
- # This test corresponds to the second test under
- # test_executable_api.md#combining-multiple-filters
- # TODO(lukasza@chromium.org): Figure out if the 3rd test example from
- # the docs has correct inputs+outputs (or if there are some typos).
- s = _SetOfTestFiltersGroups([
- _TestFiltersGroup.from_string('Foo.Bar.*'),
- _TestFiltersGroup.from_string('Foo.Baz.baz')
- ])
- self.assertEqual([], s.filter_test_names(self._tests))
- class PublicApiTests(fake_filesystem_unittest.TestCase):
- def test_filter_cmdline_arg(self):
- parser = argparse.ArgumentParser()
- test_filtering.add_cmdline_args(parser)
- args = parser.parse_args(args=[
- '--isolated-script-test-filter=-barbaz',
- '--isolated-script-test-filter=foo*::bar*'
- ])
- self.assertEqual(
- ['foo1', 'foo2', 'bar1', 'bar2'],
- test_filtering.filter_tests(
- args, {}, ['foo1', 'foo2', 'bar1', 'bar2', 'barbaz', 'zzz']))
- def test_filter_file_cmdline_arg(self):
- # pylint: disable=unexpected-keyword-arg
- f = tempfile.NamedTemporaryFile(delete=False,
- mode='w',
- encoding='utf-8')
- try:
- filepath = f.name
- f.write('foo*')
- f.close()
- parser = argparse.ArgumentParser()
- test_filtering.add_cmdline_args(parser)
- args = parser.parse_args(args=[
- '--isolated-script-test-filter-file={0:s}'.format(filepath)
- ])
- self.assertEqual(['foo1', 'foo2'],
- test_filtering.filter_tests(
- args, {}, ['foo1', 'foo2', 'bar1', 'bar2']))
- finally:
- os.remove(filepath)
- def _shard_tests(input_test_list_string, input_env):
- input_test_list = input_test_list_string.split(',')
- output_test_list = test_filtering._shard_tests(input_test_list, input_env)
- return ','.join(output_test_list)
- class ShardingTest(unittest.TestCase):
- def test_empty_environment(self):
- self.assertEqual('a,b,c', _shard_tests('a,b,c', {}))
- def test_basic_sharding(self):
- self.assertEqual(
- 'a,c,e',
- _shard_tests('a,b,c,d,e', {
- 'GTEST_SHARD_INDEX': '0',
- 'GTEST_TOTAL_SHARDS': '2'
- }))
- self.assertEqual(
- 'b,d',
- _shard_tests('a,b,c,d,e', {
- 'GTEST_SHARD_INDEX': '1',
- 'GTEST_TOTAL_SHARDS': '2'
- }))
- def test_error_conditions(self):
- # shard index > total shards
- with self.assertRaises(Exception):
- _shard_tests('', {
- 'GTEST_SHARD_INDEX': '2',
- 'GTEST_TOTAL_SHARDS': '2'
- })
- # non-integer shard index
- with self.assertRaises(Exception):
- _shard_tests('', {
- 'GTEST_SHARD_INDEX': 'a',
- 'GTEST_TOTAL_SHARDS': '2'
- })
- # non-integer total shards
- with self.assertRaises(Exception):
- _shard_tests('', {
- 'GTEST_SHARD_INDEX': '0',
- 'GTEST_TOTAL_SHARDS': 'b'
- })
- if __name__ == '__main__':
- unittest.main()
|