123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826 |
- #!/usr/bin/env vpython3
- # Copyright 2020 The Chromium Authors. All rights reserved.
- # Use of this source code is governed by a BSD-style license that can be
- # found in the LICENSE file.
- from __future__ import print_function
- import datetime
- import os
- import sys
- import tempfile
- import unittest
- if sys.version_info[0] == 2:
- import mock
- else:
- import unittest.mock as mock
- from pyfakefs import fake_filesystem_unittest
- from unexpected_passes_common import data_types
- from unexpected_passes_common import expectations
- from unexpected_passes_common import unittest_utils as uu
- FAKE_EXPECTATION_FILE_CONTENTS = """\
- # tags: [ win linux ]
- # results: [ Failure RetryOnFailure Skip Pass ]
- crbug.com/1234 [ win ] foo/test [ Failure ]
- crbug.com/5678 crbug.com/6789 [ win ] foo/another/test [ RetryOnFailure ]
- [ linux ] foo/test [ Failure ]
- crbug.com/2345 [ linux ] bar/* [ RetryOnFailure ]
- crbug.com/3456 [ linux ] some/bad/test [ Skip ]
- crbug.com/4567 [ linux ] some/good/test [ Pass ]
- """
- SECONDARY_FAKE_EXPECTATION_FILE_CONTENTS = """\
- # tags: [ mac ]
- # results: [ Failure ]
- crbug.com/4567 [ mac ] foo/test [ Failure ]
- """
- FAKE_EXPECTATION_FILE_CONTENTS_WITH_TYPO = """\
- # tags: [ win linux ]
- # results: [ Failure RetryOnFailure Skip ]
- crbug.com/1234 [ wine ] foo/test [ Failure ]
- [ linux ] foo/test [ Failure ]
- crbug.com/2345 [ linux ] bar/* [ RetryOnFailure ]
- crbug.com/3456 [ linux ] some/bad/test [ Skip ]
- """
- class CreateTestExpectationMapUnittest(unittest.TestCase):
- def setUp(self) -> None:
- self.instance = expectations.Expectations()
- self._expectation_content = {}
- self._content_patcher = mock.patch.object(
- self.instance, '_GetNonRecentExpectationContent')
- self._content_mock = self._content_patcher.start()
- self.addCleanup(self._content_patcher.stop)
- def SideEffect(filepath, _):
- return self._expectation_content[filepath]
- self._content_mock.side_effect = SideEffect
- def testExclusiveOr(self) -> None:
- """Tests that only one input can be specified."""
- with self.assertRaises(AssertionError):
- self.instance.CreateTestExpectationMap(None, None, 0)
- with self.assertRaises(AssertionError):
- self.instance.CreateTestExpectationMap('foo', ['bar'], 0)
- def testExpectationFile(self) -> None:
- """Tests reading expectations from an expectation file."""
- filename = '/tmp/foo'
- self._expectation_content[filename] = FAKE_EXPECTATION_FILE_CONTENTS
- expectation_map = self.instance.CreateTestExpectationMap(filename, None, 0)
- # Skip expectations should be omitted, but everything else should be
- # present.
- # yapf: disable
- expected_expectation_map = {
- filename: {
- data_types.Expectation(
- 'foo/test', ['win'], ['Failure'], 'crbug.com/1234'): {},
- data_types.Expectation(
- 'foo/another/test', ['win'], ['RetryOnFailure'],
- 'crbug.com/5678 crbug.com/6789'): {},
- data_types.Expectation('foo/test', ['linux'], ['Failure']): {},
- data_types.Expectation(
- 'bar/*', ['linux'], ['RetryOnFailure'], 'crbug.com/2345'): {},
- },
- }
- # yapf: enable
- self.assertEqual(expectation_map, expected_expectation_map)
- self.assertIsInstance(expectation_map, data_types.TestExpectationMap)
- def testMultipleExpectationFiles(self) -> None:
- """Tests reading expectations from multiple files."""
- filename1 = '/tmp/foo'
- filename2 = '/tmp/bar'
- expectation_files = [filename1, filename2]
- self._expectation_content[filename1] = FAKE_EXPECTATION_FILE_CONTENTS
- self._expectation_content[
- filename2] = SECONDARY_FAKE_EXPECTATION_FILE_CONTENTS
- expectation_map = self.instance.CreateTestExpectationMap(
- expectation_files, None, 0)
- # yapf: disable
- expected_expectation_map = {
- expectation_files[0]: {
- data_types.Expectation(
- 'foo/test', ['win'], ['Failure'], 'crbug.com/1234'): {},
- data_types.Expectation(
- 'foo/another/test', ['win'], ['RetryOnFailure'],
- 'crbug.com/5678 crbug.com/6789'): {},
- data_types.Expectation('foo/test', ['linux'], ['Failure']): {},
- data_types.Expectation(
- 'bar/*', ['linux'], ['RetryOnFailure'], 'crbug.com/2345'): {},
- },
- expectation_files[1]: {
- data_types.Expectation(
- 'foo/test', ['mac'], ['Failure'], 'crbug.com/4567'): {},
- }
- }
- # yapf: enable
- self.assertEqual(expectation_map, expected_expectation_map)
- self.assertIsInstance(expectation_map, data_types.TestExpectationMap)
- def testIndividualTests(self) -> None:
- """Tests reading expectations from a list of tests."""
- expectation_map = self.instance.CreateTestExpectationMap(
- None, ['foo/test', 'bar/*'], 0)
- expected_expectation_map = {
- '': {
- data_types.Expectation('foo/test', [], ['RetryOnFailure']): {},
- data_types.Expectation('bar/*', [], ['RetryOnFailure']): {},
- },
- }
- self.assertEqual(expectation_map, expected_expectation_map)
- self.assertIsInstance(expectation_map, data_types.TestExpectationMap)
- class GetNonRecentExpectationContentUnittest(unittest.TestCase):
- def setUp(self) -> None:
- self.instance = uu.CreateGenericExpectations()
- self._output_patcher = mock.patch(
- 'unexpected_passes_common.expectations.subprocess.check_output')
- self._output_mock = self._output_patcher.start()
- self.addCleanup(self._output_patcher.stop)
- def testBasic(self) -> None:
- """Tests that only expectations that are old enough are kept."""
- today_date = datetime.date.today()
- yesterday_date = today_date - datetime.timedelta(days=1)
- older_date = today_date - datetime.timedelta(days=2)
- today_str = today_date.isoformat()
- yesterday_str = yesterday_date.isoformat()
- older_str = older_date.isoformat()
- # pylint: disable=line-too-long
- blame_output = """\
- 5f03bc04975c04 (Some R. Author {today_date} 00:00:00 +0000 1)# tags: [ tag1 ]
- 98637cd80f8c15 (Some R. Author {yesterday_date} 00:00:00 +0000 2)# tags: [ tag2 ]
- 3fcadac9d861d0 (Some R. Author {older_date} 00:00:00 +0000 3)# results: [ Failure ]
- 5f03bc04975c04 (Some R. Author {today_date} 00:00:00 +0000 4)
- 5f03bc04975c04 (Some R. Author {today_date} 00:00:00 +0000 5)crbug.com/1234 [ tag1 ] testname [ Failure ]
- 98637cd80f8c15 (Some R. Author {yesterday_date} 00:00:00 +0000 6)[ tag2 ] testname [ Failure ] # Comment
- 3fcadac9d861d0 (Some R. Author {older_date} 00:00:00 +0000 7)[ tag1 ] othertest [ Failure ]"""
- # pylint: enable=line-too-long
- blame_output = blame_output.format(today_date=today_str,
- yesterday_date=yesterday_str,
- older_date=older_str)
- self._output_mock.return_value = blame_output.encode('utf-8')
- expected_content = """\
- # tags: [ tag1 ]
- # tags: [ tag2 ]
- # results: [ Failure ]
- [ tag1 ] othertest [ Failure ]"""
- self.assertEqual(self.instance._GetNonRecentExpectationContent('', 1),
- expected_content)
- def testNegativeGracePeriod(self) -> None:
- """Tests that setting a negative grace period disables filtering."""
- today_date = datetime.date.today()
- yesterday_date = today_date - datetime.timedelta(days=1)
- older_date = today_date - datetime.timedelta(days=2)
- today_str = today_date.isoformat()
- yesterday_str = yesterday_date.isoformat()
- older_str = older_date.isoformat()
- # pylint: disable=line-too-long
- blame_output = """\
- 5f03bc04975c04 (Some R. Author {today_date} 00:00:00 +0000 1)# tags: [ tag1 ]
- 98637cd80f8c15 (Some R. Author {yesterday_date} 00:00:00 +0000 2)# tags: [ tag2 ]
- 3fcadac9d861d0 (Some R. Author {older_date} 00:00:00 +0000 3)# results: [ Failure ]
- 5f03bc04975c04 (Some R. Author {today_date} 00:00:00 +0000 4)
- 5f03bc04975c04 (Some R. Author {today_date} 00:00:00 +0000 5)crbug.com/1234 [ tag1 ] testname [ Failure ]
- 98637cd80f8c15 (Some R. Author {yesterday_date} 00:00:00 +0000 6)[ tag2 ] testname [ Failure ] # Comment
- 3fcadac9d861d0 (Some R. Author {older_date} 00:00:00 +0000 7)[ tag1 ] othertest [ Failure ]"""
- # pylint: enable=line-too-long
- blame_output = blame_output.format(today_date=today_str,
- yesterday_date=yesterday_str,
- older_date=older_str)
- self._output_mock.return_value = blame_output.encode('utf-8')
- expected_content = """\
- # tags: [ tag1 ]
- # tags: [ tag2 ]
- # results: [ Failure ]
- crbug.com/1234 [ tag1 ] testname [ Failure ]
- [ tag2 ] testname [ Failure ] # Comment
- [ tag1 ] othertest [ Failure ]"""
- self.assertEqual(self.instance._GetNonRecentExpectationContent('', -1),
- expected_content)
- class RemoveExpectationsFromFileUnittest(fake_filesystem_unittest.TestCase):
- def setUp(self) -> None:
- self.instance = uu.CreateGenericExpectations()
- self.header = self.instance._GetExpectationFileTagHeader(None)
- self.setUpPyfakefs()
- with tempfile.NamedTemporaryFile(delete=False) as f:
- self.filename = f.name
- def testExpectationRemoval(self) -> None:
- """Tests that expectations are properly removed from a file."""
- contents = self.header + """
- # This is a test comment
- crbug.com/1234 [ win ] foo/test [ Failure ]
- crbug.com/2345 [ win ] foo/test [ RetryOnFailure ]
- # Another comment
- [ linux ] bar/test [ RetryOnFailure ]
- [ win ] bar/test [ RetryOnFailure ]
- """
- stale_expectations = [
- data_types.Expectation('foo/test', ['win'], ['Failure'],
- 'crbug.com/1234'),
- data_types.Expectation('bar/test', ['linux'], ['RetryOnFailure'])
- ]
- expected_contents = self.header + """
- # This is a test comment
- crbug.com/2345 [ win ] foo/test [ RetryOnFailure ]
- # Another comment
- [ win ] bar/test [ RetryOnFailure ]
- """
- with open(self.filename, 'w') as f:
- f.write(contents)
- removed_urls = self.instance.RemoveExpectationsFromFile(
- stale_expectations, self.filename, expectations.RemovalType.STALE)
- self.assertEqual(removed_urls, set(['crbug.com/1234']))
- with open(self.filename) as f:
- self.assertEqual(f.read(), expected_contents)
- def testRemovalWithMultipleBugs(self) -> None:
- """Tests removal of expectations with multiple associated bugs."""
- contents = self.header + """
- # This is a test comment
- crbug.com/1234 crbug.com/3456 crbug.com/4567 [ win ] foo/test [ Failure ]
- crbug.com/2345 [ win ] foo/test [ RetryOnFailure ]
- # Another comment
- [ linux ] bar/test [ RetryOnFailure ]
- [ win ] bar/test [ RetryOnFailure ]
- """
- stale_expectations = [
- data_types.Expectation('foo/test', ['win'], ['Failure'],
- 'crbug.com/1234 crbug.com/3456 crbug.com/4567'),
- ]
- expected_contents = self.header + """
- # This is a test comment
- crbug.com/2345 [ win ] foo/test [ RetryOnFailure ]
- # Another comment
- [ linux ] bar/test [ RetryOnFailure ]
- [ win ] bar/test [ RetryOnFailure ]
- """
- with open(self.filename, 'w') as f:
- f.write(contents)
- removed_urls = self.instance.RemoveExpectationsFromFile(
- stale_expectations, self.filename, expectations.RemovalType.STALE)
- self.assertEqual(
- removed_urls,
- set(['crbug.com/1234', 'crbug.com/3456', 'crbug.com/4567']))
- with open(self.filename) as f:
- self.assertEqual(f.read(), expected_contents)
- def testNestedBlockComments(self) -> None:
- """Tests that nested disable block comments throw exceptions."""
- contents = self.header + """
- # finder:disable-general
- # finder:disable-general
- crbug.com/1234 [ win ] foo/test [ Failure ]
- # finder:enable-general
- # finder:enable-general
- """
- with open(self.filename, 'w') as f:
- f.write(contents)
- with self.assertRaises(RuntimeError):
- self.instance.RemoveExpectationsFromFile([], self.filename,
- expectations.RemovalType.STALE)
- contents = self.header + """
- # finder:disable-general
- # finder:disable-stale
- crbug.com/1234 [ win ] foo/test [ Failure ]
- # finder:enable-stale
- # finder:enable-genearl
- """
- with open(self.filename, 'w') as f:
- f.write(contents)
- with self.assertRaises(RuntimeError):
- self.instance.RemoveExpectationsFromFile([], self.filename,
- expectations.RemovalType.STALE)
- contents = self.header + """
- # finder:enable-general
- crbug.com/1234 [ win ] foo/test [ Failure ]
- """
- with open(self.filename, 'w') as f:
- f.write(contents)
- with self.assertRaises(RuntimeError):
- self.instance.RemoveExpectationsFromFile([], self.filename,
- expectations.RemovalType.STALE)
- def testGeneralBlockComments(self) -> None:
- """Tests that expectations in a disable block comment are not removed."""
- contents = self.header + """
- crbug.com/1234 [ win ] foo/test [ Failure ]
- # finder:disable-general
- crbug.com/2345 [ win ] foo/test [ Failure ]
- crbug.com/3456 [ win ] foo/test [ Failure ]
- # finder:enable-general
- crbug.com/4567 [ win ] foo/test [ Failure ]
- """
- stale_expectations = [
- data_types.Expectation('foo/test', ['win'], ['Failure'],
- 'crbug.com/1234'),
- data_types.Expectation('foo/test', ['win'], ['Failure'],
- 'crbug.com/2345'),
- data_types.Expectation('foo/test', ['win'], ['Failure'],
- 'crbug.com/3456'),
- data_types.Expectation('foo/test', ['win'], ['Failure'],
- 'crbug.com/4567'),
- ]
- expected_contents = self.header + """
- # finder:disable-general
- crbug.com/2345 [ win ] foo/test [ Failure ]
- crbug.com/3456 [ win ] foo/test [ Failure ]
- # finder:enable-general
- """
- for removal_type in (expectations.RemovalType.STALE,
- expectations.RemovalType.UNUSED):
- with open(self.filename, 'w') as f:
- f.write(contents)
- removed_urls = self.instance.RemoveExpectationsFromFile(
- stale_expectations, self.filename, removal_type)
- self.assertEqual(removed_urls, set(['crbug.com/1234', 'crbug.com/4567']))
- with open(self.filename) as f:
- self.assertEqual(f.read(), expected_contents)
- def testStaleBlockComments(self) -> None:
- """Tests that stale expectations in a stale disable block are not removed"""
- contents = self.header + """
- crbug.com/1234 [ win ] not_stale [ Failure ]
- crbug.com/1234 [ win ] before_block [ Failure ]
- # finder:disable-stale
- crbug.com/2345 [ win ] in_block [ Failure ]
- # finder:enable-stale
- crbug.com/3456 [ win ] after_block [ Failure ]
- """
- stale_expectations = [
- data_types.Expectation('before_block', ['win'], 'Failure',
- 'crbug.com/1234'),
- data_types.Expectation('in_block', ['win'], 'Failure',
- 'crbug.com/2345'),
- data_types.Expectation('after_block', ['win'], 'Failure',
- 'crbug.com/3456'),
- ]
- expected_contents = self.header + """
- crbug.com/1234 [ win ] not_stale [ Failure ]
- # finder:disable-stale
- crbug.com/2345 [ win ] in_block [ Failure ]
- # finder:enable-stale
- """
- with open(self.filename, 'w') as f:
- f.write(contents)
- removed_urls = self.instance.RemoveExpectationsFromFile(
- stale_expectations, self.filename, expectations.RemovalType.STALE)
- self.assertEqual(removed_urls, set(['crbug.com/1234', 'crbug.com/3456']))
- with open(self.filename) as f:
- self.assertEqual(f.read(), expected_contents)
- def testUnusedBlockComments(self) -> None:
- """Tests that stale expectations in unused disable blocks are not removed"""
- contents = self.header + """
- crbug.com/1234 [ win ] not_unused [ Failure ]
- crbug.com/1234 [ win ] before_block [ Failure ]
- # finder:disable-unused
- crbug.com/2345 [ win ] in_block [ Failure ]
- # finder:enable-unused
- crbug.com/3456 [ win ] after_block [ Failure ]
- """
- unused_expectations = [
- data_types.Expectation('before_block', ['win'], 'Failure',
- 'crbug.com/1234'),
- data_types.Expectation('in_block', ['win'], 'Failure',
- 'crbug.com/2345'),
- data_types.Expectation('after_block', ['win'], 'Failure',
- 'crbug.com/3456'),
- ]
- expected_contents = self.header + """
- crbug.com/1234 [ win ] not_unused [ Failure ]
- # finder:disable-unused
- crbug.com/2345 [ win ] in_block [ Failure ]
- # finder:enable-unused
- """
- with open(self.filename, 'w') as f:
- f.write(contents)
- removed_urls = self.instance.RemoveExpectationsFromFile(
- unused_expectations, self.filename, expectations.RemovalType.UNUSED)
- self.assertEqual(removed_urls, set(['crbug.com/1234', 'crbug.com/3456']))
- with open(self.filename) as f:
- self.assertEqual(f.read(), expected_contents)
- def testMismatchedBlockComments(self) -> None:
- """Tests that block comments for the wrong removal type do nothing."""
- contents = self.header + """
- crbug.com/1234 [ win ] do_not_remove [ Failure ]
- # finder:disable-stale
- crbug.com/2345 [ win ] disabled_stale [ Failure ]
- # finder:enable-stale
- # finder:disable-unused
- crbug.com/3456 [ win ] disabled_unused [ Failure ]
- # finder:enable-unused
- crbug.com/4567 [ win ] also_do_not_remove [ Failure ]
- """
- expectations_to_remove = [
- data_types.Expectation('disabled_stale', ['win'], 'Failure',
- 'crbug.com/2345'),
- data_types.Expectation('disabled_unused', ['win'], 'Failure',
- 'crbug.com/3456'),
- ]
- expected_contents = self.header + """
- crbug.com/1234 [ win ] do_not_remove [ Failure ]
- # finder:disable-stale
- crbug.com/2345 [ win ] disabled_stale [ Failure ]
- # finder:enable-stale
- # finder:disable-unused
- # finder:enable-unused
- crbug.com/4567 [ win ] also_do_not_remove [ Failure ]
- """
- with open(self.filename, 'w') as f:
- f.write(contents)
- removed_urls = self.instance.RemoveExpectationsFromFile(
- expectations_to_remove, self.filename, expectations.RemovalType.STALE)
- self.assertEqual(removed_urls, set(['crbug.com/3456']))
- with open(self.filename) as f:
- self.assertEqual(f.read(), expected_contents)
- expected_contents = self.header + """
- crbug.com/1234 [ win ] do_not_remove [ Failure ]
- # finder:disable-stale
- # finder:enable-stale
- # finder:disable-unused
- crbug.com/3456 [ win ] disabled_unused [ Failure ]
- # finder:enable-unused
- crbug.com/4567 [ win ] also_do_not_remove [ Failure ]
- """
- with open(self.filename, 'w') as f:
- f.write(contents)
- removed_urls = self.instance.RemoveExpectationsFromFile(
- expectations_to_remove, self.filename, expectations.RemovalType.UNUSED)
- self.assertEqual(removed_urls, set(['crbug.com/2345']))
- with open(self.filename) as f:
- self.assertEqual(f.read(), expected_contents)
- def testInlineGeneralComments(self) -> None:
- """Tests that expectations with inline disable comments are not removed."""
- contents = self.header + """
- crbug.com/1234 [ win ] foo/test [ Failure ]
- crbug.com/2345 [ win ] foo/test [ Failure ] # finder:disable-general
- crbug.com/3456 [ win ] foo/test [ Failure ]
- """
- stale_expectations = [
- data_types.Expectation('foo/test', ['win'], ['Failure'],
- 'crbug.com/1234'),
- data_types.Expectation('foo/test', ['win'], ['Failure'],
- 'crbug.com/2345'),
- data_types.Expectation('foo/test', ['win'], ['Failure'],
- 'crbug.com/3456'),
- ]
- expected_contents = self.header + """
- crbug.com/2345 [ win ] foo/test [ Failure ] # finder:disable-general
- """
- for removal_type in (expectations.RemovalType.STALE,
- expectations.RemovalType.UNUSED):
- with open(self.filename, 'w') as f:
- f.write(contents)
- removed_urls = self.instance.RemoveExpectationsFromFile(
- stale_expectations, self.filename, removal_type)
- self.assertEqual(removed_urls, set(['crbug.com/1234', 'crbug.com/3456']))
- with open(self.filename) as f:
- self.assertEqual(f.read(), expected_contents)
- def testInlineStaleComments(self) -> None:
- """Tests that expectations with inline stale disable comments not removed"""
- contents = self.header + """
- crbug.com/1234 [ win ] not_disabled [ Failure ]
- crbug.com/2345 [ win ] stale_disabled [ Failure ] # finder:disable-stale
- crbug.com/3456 [ win ] unused_disabled [ Failure ] # finder:disable-unused
- """
- stale_expectations = [
- data_types.Expectation('not_disabled', ['win'], 'Failure',
- 'crbug.com/1234'),
- data_types.Expectation('stale_disabled', ['win'], 'Failure',
- 'crbug.com/2345'),
- data_types.Expectation('unused_disabled', ['win'], 'Failure',
- 'crbug.com/3456')
- ]
- expected_contents = self.header + """
- crbug.com/2345 [ win ] stale_disabled [ Failure ] # finder:disable-stale
- """
- with open(self.filename, 'w') as f:
- f.write(contents)
- removed_urls = self.instance.RemoveExpectationsFromFile(
- stale_expectations, self.filename, expectations.RemovalType.STALE)
- self.assertEqual(removed_urls, set(['crbug.com/1234', 'crbug.com/3456']))
- with open(self.filename) as f:
- self.assertEqual(f.read(), expected_contents)
- def testInlineUnusedComments(self) -> None:
- """Tests that expectations with inline unused comments not removed"""
- contents = self.header + """
- crbug.com/1234 [ win ] not_disabled [ Failure ]
- crbug.com/2345 [ win ] stale_disabled [ Failure ] # finder:disable-stale
- crbug.com/3456 [ win ] unused_disabled [ Failure ] # finder:disable-unused
- """
- stale_expectations = [
- data_types.Expectation('not_disabled', ['win'], 'Failure',
- 'crbug.com/1234'),
- data_types.Expectation('stale_disabled', ['win'], 'Failure',
- 'crbug.com/2345'),
- data_types.Expectation('unused_disabled', ['win'], 'Failure',
- 'crbug.com/3456')
- ]
- expected_contents = self.header + """
- crbug.com/3456 [ win ] unused_disabled [ Failure ] # finder:disable-unused
- """
- with open(self.filename, 'w') as f:
- f.write(contents)
- removed_urls = self.instance.RemoveExpectationsFromFile(
- stale_expectations, self.filename, expectations.RemovalType.UNUSED)
- self.assertEqual(removed_urls, set(['crbug.com/1234', 'crbug.com/2345']))
- with open(self.filename) as f:
- self.assertEqual(f.read(), expected_contents)
- def testGetDisableReasonFromComment(self):
- """Tests that the disable reason can be pulled from a line."""
- self.assertEqual(
- expectations._GetDisableReasonFromComment(
- '# finder:disable-general foo'), 'foo')
- self.assertEqual(
- expectations._GetDisableReasonFromComment(
- 'crbug.com/1234 [ win ] bar/test [ Failure ] '
- '# finder:disable-general foo'), 'foo')
- class GetExpectationLineUnittest(unittest.TestCase):
- def setUp(self) -> None:
- self.instance = uu.CreateGenericExpectations()
- def testNoMatchingExpectation(self) -> None:
- """Tests that the case of no matching expectation is handled."""
- expectation = data_types.Expectation('foo', ['win'], 'Failure')
- line, line_number = self.instance._GetExpectationLine(
- expectation, FAKE_EXPECTATION_FILE_CONTENTS, 'expectation_file')
- self.assertIsNone(line)
- self.assertIsNone(line_number)
- def testMatchingExpectation(self) -> None:
- """Tests that matching expectations are found."""
- expectation = data_types.Expectation('foo/test', ['win'], 'Failure',
- 'crbug.com/1234')
- line, line_number = self.instance._GetExpectationLine(
- expectation, FAKE_EXPECTATION_FILE_CONTENTS, 'expectation_file')
- self.assertEqual(line, 'crbug.com/1234 [ win ] foo/test [ Failure ]')
- self.assertEqual(line_number, 3)
- class ModifySemiStaleExpectationsUnittest(fake_filesystem_unittest.TestCase):
- def setUp(self) -> None:
- self.setUpPyfakefs()
- self.instance = uu.CreateGenericExpectations()
- self._input_patcher = mock.patch.object(expectations,
- '_WaitForUserInputOnModification')
- self._input_mock = self._input_patcher.start()
- self.addCleanup(self._input_patcher.stop)
- with tempfile.NamedTemporaryFile(delete=False, mode='w') as f:
- f.write(FAKE_EXPECTATION_FILE_CONTENTS)
- self.filename = f.name
- with tempfile.NamedTemporaryFile(delete=False, mode='w') as f:
- f.write(SECONDARY_FAKE_EXPECTATION_FILE_CONTENTS)
- self.secondary_filename = f.name
- def testEmptyExpectationMap(self) -> None:
- """Tests that an empty expectation map results in a no-op."""
- modified_urls = self.instance.ModifySemiStaleExpectations(
- data_types.TestExpectationMap())
- self.assertEqual(modified_urls, set())
- self._input_mock.assert_not_called()
- with open(self.filename) as f:
- self.assertEqual(f.read(), FAKE_EXPECTATION_FILE_CONTENTS)
- def testRemoveExpectation(self) -> None:
- """Tests that specifying to remove an expectation does so."""
- self._input_mock.return_value = 'r'
- # yapf: disable
- test_expectation_map = data_types.TestExpectationMap({
- self.filename:
- data_types.ExpectationBuilderMap({
- data_types.Expectation(
- 'foo/test', ['win'], 'Failure', 'crbug.com/1234'):
- data_types.BuilderStepMap(),
- }),
- self.secondary_filename:
- data_types.ExpectationBuilderMap({
- data_types.Expectation(
- 'foo/test', ['mac'], 'Failure', 'crbug.com/4567'):
- data_types.BuilderStepMap(),
- }),
- })
- # yapf: enable
- modified_urls = self.instance.ModifySemiStaleExpectations(
- test_expectation_map)
- self.assertEqual(modified_urls, set(['crbug.com/1234', 'crbug.com/4567']))
- expected_file_contents = """\
- # tags: [ win linux ]
- # results: [ Failure RetryOnFailure Skip Pass ]
- crbug.com/5678 crbug.com/6789 [ win ] foo/another/test [ RetryOnFailure ]
- [ linux ] foo/test [ Failure ]
- crbug.com/2345 [ linux ] bar/* [ RetryOnFailure ]
- crbug.com/3456 [ linux ] some/bad/test [ Skip ]
- crbug.com/4567 [ linux ] some/good/test [ Pass ]
- """
- with open(self.filename) as f:
- self.assertEqual(f.read(), expected_file_contents)
- expected_file_contents = """\
- # tags: [ mac ]
- # results: [ Failure ]
- """
- with open(self.secondary_filename) as f:
- self.assertEqual(f.read(), expected_file_contents)
- def testModifyExpectation(self) -> None:
- """Tests that specifying to modify an expectation does not remove it."""
- self._input_mock.return_value = 'm'
- # yapf: disable
- test_expectation_map = data_types.TestExpectationMap({
- self.filename:
- data_types.ExpectationBuilderMap({
- data_types.Expectation(
- 'foo/test', ['win'], 'Failure', 'crbug.com/1234'):
- data_types.BuilderStepMap(),
- }),
- self.secondary_filename:
- data_types.ExpectationBuilderMap({
- data_types.Expectation(
- 'foo/test', ['mac'], 'Failure', 'crbug.com/4567',
- ): data_types.BuilderStepMap()
- }),
- })
- # yapf: enable
- modified_urls = self.instance.ModifySemiStaleExpectations(
- test_expectation_map)
- self.assertEqual(modified_urls, set(['crbug.com/1234', 'crbug.com/4567']))
- with open(self.filename) as f:
- self.assertEqual(f.read(), FAKE_EXPECTATION_FILE_CONTENTS)
- with open(self.secondary_filename) as f:
- self.assertEqual(f.read(), SECONDARY_FAKE_EXPECTATION_FILE_CONTENTS)
- def testModifyExpectationMultipleBugs(self) -> None:
- """Tests that modifying an expectation with multiple bugs works properly."""
- self._input_mock.return_value = 'm'
- # yapf: disable
- test_expectation_map = data_types.TestExpectationMap({
- self.filename:
- data_types.ExpectationBuilderMap({
- data_types.Expectation(
- 'foo/another/test', ['win'], 'RetryOnFailure',
- 'crbug.com/5678 crbug.com/6789'):
- data_types.BuilderStepMap(),
- }),
- })
- # yapf: enable
- modified_urls = self.instance.ModifySemiStaleExpectations(
- test_expectation_map)
- self.assertEqual(modified_urls, set(['crbug.com/5678', 'crbug.com/6789']))
- with open(self.filename) as f:
- self.assertEqual(f.read(), FAKE_EXPECTATION_FILE_CONTENTS)
- with open(self.secondary_filename) as f:
- self.assertEqual(f.read(), SECONDARY_FAKE_EXPECTATION_FILE_CONTENTS)
- def testIgnoreExpectation(self) -> None:
- """Tests that specifying to ignore an expectation does nothing."""
- self._input_mock.return_value = 'i'
- # yapf: disable
- test_expectation_map = data_types.TestExpectationMap({
- self.filename:
- data_types.ExpectationBuilderMap({
- data_types.Expectation(
- 'foo/test', ['win'], 'Failure', 'crbug.com/1234'):
- data_types.BuilderStepMap(),
- }),
- self.secondary_filename:
- data_types.ExpectationBuilderMap({
- data_types.Expectation(
- 'foo/test', ['mac'], 'Failure', 'crbug.com/4567',
- ): data_types.BuilderStepMap()
- }),
- })
- # yapf: enable
- modified_urls = self.instance.ModifySemiStaleExpectations(
- test_expectation_map)
- self.assertEqual(modified_urls, set())
- with open(self.filename) as f:
- self.assertEqual(f.read(), FAKE_EXPECTATION_FILE_CONTENTS)
- with open(self.secondary_filename) as f:
- self.assertEqual(f.read(), SECONDARY_FAKE_EXPECTATION_FILE_CONTENTS)
- def testParserErrorCorrection(self) -> None:
- """Tests that parser errors are caught and users can fix them."""
- def TypoSideEffect() -> str:
- with open(self.filename, 'w') as outfile:
- outfile.write(FAKE_EXPECTATION_FILE_CONTENTS_WITH_TYPO)
- return 'm'
- def CorrectionSideEffect() -> None:
- with open(self.filename, 'w') as outfile:
- outfile.write(FAKE_EXPECTATION_FILE_CONTENTS)
- self._input_mock.side_effect = TypoSideEffect
- with mock.patch.object(expectations,
- '_WaitForAnyUserInput') as any_input_mock:
- any_input_mock.side_effect = CorrectionSideEffect
- # yapf: disable
- test_expectation_map = data_types.TestExpectationMap({
- self.filename:
- data_types.ExpectationBuilderMap({
- data_types.Expectation(
- 'foo/test', ['win'], 'Failure', 'crbug.com/1234'):
- data_types.BuilderStepMap(),
- }),
- })
- # yapf: enable
- self.instance.ModifySemiStaleExpectations(test_expectation_map)
- any_input_mock.assert_called_once()
- with open(self.filename) as infile:
- self.assertEqual(infile.read(), FAKE_EXPECTATION_FILE_CONTENTS)
- class FindOrphanedBugsUnittest(fake_filesystem_unittest.TestCase):
- def CreateFile(self, *args, **kwargs) -> None:
- # TODO(crbug.com/1156806): Remove this and just use fs.create_file() when
- # Catapult is updated to a newer version of pyfakefs that is compatible with
- # Chromium's version.
- if hasattr(self.fs, 'create_file'):
- self.fs.create_file(*args, **kwargs)
- else:
- self.fs.CreateFile(*args, **kwargs)
- def setUp(self) -> None:
- expectations_dir = os.path.join(os.path.dirname(__file__), 'expectations')
- self.setUpPyfakefs()
- self.instance = expectations.Expectations()
- self.filepath_patcher = mock.patch.object(
- self.instance,
- 'GetExpectationFilepaths',
- return_value=[os.path.join(expectations_dir, 'real_expectations.txt')])
- self.filepath_mock = self.filepath_patcher.start()
- self.addCleanup(self.filepath_patcher.stop)
- real_contents = 'crbug.com/1\ncrbug.com/2'
- skipped_contents = 'crbug.com/4'
- self.CreateFile(os.path.join(expectations_dir, 'real_expectations.txt'),
- contents=real_contents)
- self.CreateFile(os.path.join(expectations_dir, 'fake.txt'),
- contents=skipped_contents)
- def testNoOrphanedBugs(self) -> None:
- bugs = ['crbug.com/1', 'crbug.com/2']
- self.assertEqual(self.instance.FindOrphanedBugs(bugs), set())
- def testOrphanedBugs(self) -> None:
- bugs = ['crbug.com/1', 'crbug.com/3', 'crbug.com/4']
- self.assertEqual(self.instance.FindOrphanedBugs(bugs),
- set(['crbug.com/3', 'crbug.com/4']))
- if __name__ == '__main__':
- unittest.main(verbosity=2)
|