123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589 |
- #!/usr/bin/env vpython3
- # Copyright 2020 The Chromium Authors. All rights reserved.
- # Use of this source code is governed by a BSD-style license that can be
- # found in the LICENSE file.
- import copy
- import json
- import subprocess
- import sys
- from typing import List, Tuple
- import unittest
- import unittest.mock as mock
- from unexpected_passes_common import builders
- from unexpected_passes_common import constants
- from unexpected_passes_common import data_types
- from unexpected_passes_common import multiprocessing_utils
- from unexpected_passes_common import queries
- from unexpected_passes_common import unittest_utils
- queries.QUERY_DELAY = 0
- class HelperMethodUnittest(unittest.TestCase):
- def testStripPrefixFromBuildIdValidId(self) -> None:
- self.assertEqual(queries._StripPrefixFromBuildId('build-1'), '1')
- def testStripPrefixFromBuildIdInvalidId(self) -> None:
- with self.assertRaises(AssertionError):
- queries._StripPrefixFromBuildId('build1')
- with self.assertRaises(AssertionError):
- queries._StripPrefixFromBuildId('build-1-2')
- def testConvertActualResultToExpectationFileFormatAbort(self) -> None:
- self.assertEqual(
- queries._ConvertActualResultToExpectationFileFormat('ABORT'), 'Timeout')
- class QueryGeneratorUnittest(unittest.TestCase):
- def setUp(self):
- self._builder = data_types.BuilderEntry('ci', constants.BuilderTypes.CI,
- False)
- def testSplitQueryGeneratorInitialSplit(self) -> None:
- """Tests that initial query splitting works as expected."""
- test_filter = queries.SplitQueryGenerator(self._builder, ['1', '2', '3'], 2)
- self.assertEqual(test_filter._test_id_lists, [['1', '2'], ['3']])
- self.assertEqual(len(test_filter.GetClauses()), 2)
- test_filter = queries.SplitQueryGenerator(self._builder, ['1', '2', '3'], 3)
- self.assertEqual(test_filter._test_id_lists, [['1', '2', '3']])
- self.assertEqual(len(test_filter.GetClauses()), 1)
- def testSplitQueryGeneratorSplitQuery(self) -> None:
- """Tests that SplitQueryGenerator's query splitting works."""
- test_filter = queries.SplitQueryGenerator(self._builder, ['1', '2'], 10)
- self.assertEqual(len(test_filter.GetClauses()), 1)
- test_filter.SplitQuery()
- self.assertEqual(len(test_filter.GetClauses()), 2)
- def testSplitQueryGeneratorSplitQueryCannotSplitFurther(self) -> None:
- """Tests that SplitQueryGenerator's failure mode."""
- test_filter = queries.SplitQueryGenerator(self._builder, ['1'], 1)
- with self.assertRaises(queries.QuerySplitError):
- test_filter.SplitQuery()
- class QueryBuilderUnittest(unittest.TestCase):
- def setUp(self) -> None:
- self._patcher = mock.patch.object(subprocess, 'Popen')
- self._popen_mock = self._patcher.start()
- self.addCleanup(self._patcher.stop)
- builders.ClearInstance()
- unittest_utils.RegisterGenericBuildersImplementation()
- self._querier = unittest_utils.CreateGenericQuerier()
- self._relevant_file_patcher = mock.patch.object(
- self._querier,
- '_GetRelevantExpectationFilesForQueryResult',
- return_value=None)
- self._relevant_file_mock = self._relevant_file_patcher.start()
- self.addCleanup(self._relevant_file_patcher.stop)
- self._builder = data_types.BuilderEntry('builder',
- constants.BuilderTypes.CI, False)
- def testQueryFailureRaised(self) -> None:
- """Tests that a query failure is properly surfaced."""
- self._popen_mock.return_value = unittest_utils.FakeProcess(returncode=1)
- with self.assertRaises(RuntimeError):
- self._querier.QueryBuilder(
- data_types.BuilderEntry('builder', constants.BuilderTypes.CI, False))
- def testInvalidNumSamples(self) -> None:
- """Tests that the number of samples is validated."""
- with self.assertRaises(AssertionError):
- unittest_utils.CreateGenericQuerier(num_samples=-1)
- def testNoResults(self) -> None:
- """Tests functionality if the query returns no results."""
- self._popen_mock.return_value = unittest_utils.FakeProcess(stdout='[]')
- results, expectation_files = self._querier.QueryBuilder(
- data_types.BuilderEntry('builder', constants.BuilderTypes.CI, False))
- self.assertEqual(results, [])
- self.assertIsNone(expectation_files, None)
- def testValidResults(self) -> None:
- """Tests functionality when valid results are returned."""
- self._relevant_file_mock.return_value = ['foo_expectations']
- query_results = [
- {
- 'id':
- 'build-1234',
- 'test_id': ('ninja://chrome/test:telemetry_gpu_integration_test/'
- 'gpu_tests.pixel_integration_test.'
- 'PixelIntegrationTest.test_name'),
- 'status':
- 'FAIL',
- 'typ_expectations': [
- 'RetryOnFailure',
- ],
- 'typ_tags': [
- 'win',
- 'intel',
- ],
- 'step_name':
- 'step_name',
- },
- ]
- self._popen_mock.return_value = unittest_utils.FakeProcess(
- stdout=json.dumps(query_results))
- results, expectation_files = self._querier.QueryBuilder(
- data_types.BuilderEntry('builder', constants.BuilderTypes.CI, False))
- self.assertEqual(len(results), 1)
- self.assertEqual(
- results[0],
- data_types.Result('test_name', ['win', 'intel'], 'Failure', 'step_name',
- '1234'))
- self.assertEqual(expectation_files, ['foo_expectations'])
- def testValidResultsNoneExpectations(self) -> None:
- """Tests when an implementation uses None for expectation files."""
- query_results = [
- {
- 'id':
- 'build-1234',
- 'test_id': ('ninja://chrome/test:telemetry_gpu_integration_test/'
- 'gpu_tests.pixel_integration_test.'
- 'PixelIntegrationTest.test_name'),
- 'status':
- 'FAIL',
- 'typ_expectations': [
- 'RetryOnFailure',
- ],
- 'typ_tags': [
- 'win',
- 'intel',
- ],
- 'step_name':
- 'step_name',
- },
- {
- 'id':
- 'build-1234',
- 'test_id': ('ninja://chrome/test:telemetry_gpu_integration_test/'
- 'gpu_tests.pixel_integration_test.'
- 'PixelIntegrationTest.test_name'),
- 'status':
- 'FAIL',
- 'typ_expectations': [
- 'RetryOnFailure',
- ],
- 'typ_tags': [
- 'win',
- 'nvidia',
- ],
- 'step_name':
- 'step_name',
- },
- ]
- self._popen_mock.return_value = unittest_utils.FakeProcess(
- stdout=json.dumps(query_results))
- with mock.patch.object(
- self._querier, '_GetRelevantExpectationFilesForQueryResult') as ef_mock:
- ef_mock.return_value = None
- results, expectation_files = self._querier.QueryBuilder(
- data_types.BuilderEntry('builder', constants.BuilderTypes.CI, False))
- self.assertEqual(len(results), 2)
- self.assertIn(
- data_types.Result('test_name', ['win', 'intel'], 'Failure',
- 'step_name', '1234'), results)
- self.assertIn(
- data_types.Result('test_name', ['win', 'nvidia'], 'Failure',
- 'step_name', '1234'), results)
- self.assertIsNone(expectation_files)
- ef_mock.assert_called_once()
- def testValidResultsMultipleSteps(self) -> None:
- """Tests functionality when results from multiple steps are present."""
- def SideEffect(result: queries.QueryResult) -> List[str]:
- if result['step_name'] == 'a step name':
- return ['foo_expectations']
- if result['step_name'] == 'another step name':
- return ['bar_expectations']
- raise RuntimeError('Unknown step %s' % result['step_name'])
- self._relevant_file_mock.side_effect = SideEffect
- query_results = [
- {
- 'id': 'build-1234',
- 'test_id': 'ninja://:blink_web_tests/some/test/with.test_name',
- 'status': 'FAIL',
- 'typ_expectations': [
- 'Failure',
- ],
- 'typ_tags': [
- 'linux',
- 'release',
- ],
- 'step_name': 'a step name',
- },
- {
- 'id': 'build-1234',
- 'test_id': 'ninja://:blink_web_tests/some/test/with.test_name',
- 'status': 'FAIL',
- 'typ_expectations': [
- 'Crash',
- ],
- 'typ_tags': [
- 'linux',
- 'debug',
- ],
- 'step_name': 'another step name',
- },
- ]
- self._popen_mock.return_value = unittest_utils.FakeProcess(
- stdout=json.dumps(query_results))
- results, expectation_files = self._querier.QueryBuilder(
- data_types.BuilderEntry('builder', constants.BuilderTypes.CI, False))
- self.assertEqual(len(results), 2)
- self.assertIn(
- data_types.Result('test_name', ['linux', 'release'], 'Failure',
- 'a step name', '1234'), results)
- self.assertIn(
- data_types.Result('test_name', ['linux', 'debug'], 'Failure',
- 'another step name', '1234'), results)
- self.assertEqual(len(expectation_files), 2)
- self.assertEqual(set(expectation_files),
- set(['foo_expectations', 'bar_expectations']))
- def testFilterInsertion(self) -> None:
- """Tests that test filters are properly inserted into the query."""
- with mock.patch.object(
- self._querier,
- '_GetQueryGeneratorForBuilder',
- return_value=unittest_utils.SimpleFixedQueryGenerator(
- self._builder, 'a real filter')), mock.patch.object(
- self._querier,
- '_RunBigQueryCommandsForJsonOutput') as query_mock:
- self._querier.QueryBuilder(self._builder)
- query_mock.assert_called_once()
- query = query_mock.call_args[0][0][0]
- self.assertIn('a real filter', query)
- def testEarlyReturnOnNoFilter(self) -> None:
- """Tests that the absence of a test filter results in an early return."""
- with mock.patch.object(
- self._querier, '_GetQueryGeneratorForBuilder',
- return_value=None), mock.patch.object(
- self._querier, '_RunBigQueryCommandsForJsonOutput') as query_mock:
- results, expectation_files = self._querier.QueryBuilder(self._builder)
- query_mock.assert_not_called()
- self.assertEqual(results, [])
- self.assertEqual(expectation_files, None)
- def testRetryOnMemoryLimit(self) -> None:
- """Tests that queries are split and retried if the memory limit is hit."""
- def SideEffect(*_, **__) -> list:
- SideEffect.call_count += 1
- if SideEffect.call_count == 1:
- raise queries.MemoryLimitError()
- return []
- SideEffect.call_count = 0
- with mock.patch.object(
- self._querier,
- '_GetQueryGeneratorForBuilder',
- return_value=unittest_utils.SimpleSplitQueryGenerator(
- self._builder, ['filter_a', 'filter_b'], 10)), mock.patch.object(
- self._querier,
- '_RunBigQueryCommandsForJsonOutput') as query_mock:
- query_mock.side_effect = SideEffect
- self._querier.QueryBuilder(self._builder)
- self.assertEqual(query_mock.call_count, 2)
- args, _ = unittest_utils.GetArgsForMockCall(query_mock.call_args_list, 0)
- first_query = args[0][0]
- self.assertIn('filter_a', first_query)
- self.assertIn('filter_b', first_query)
- args, _ = unittest_utils.GetArgsForMockCall(query_mock.call_args_list, 1)
- second_query_first_half = args[0][0]
- self.assertIn('filter_a', second_query_first_half)
- self.assertNotIn('filter_b', second_query_first_half)
- second_query_second_half = args[0][1]
- self.assertIn('filter_b', second_query_second_half)
- self.assertNotIn('filter_a', second_query_second_half)
- class FillExpectationMapForBuildersUnittest(unittest.TestCase):
- def setUp(self) -> None:
- self._querier = unittest_utils.CreateGenericQuerier()
- self._query_patcher = mock.patch.object(self._querier, 'QueryBuilder')
- self._query_mock = self._query_patcher.start()
- self.addCleanup(self._query_patcher.stop)
- self._pool_patcher = mock.patch.object(multiprocessing_utils,
- 'GetProcessPool')
- self._pool_mock = self._pool_patcher.start()
- self._pool_mock.return_value = unittest_utils.FakePool()
- self.addCleanup(self._pool_patcher.stop)
- self._filter_patcher = mock.patch.object(self._querier,
- '_FilterOutInactiveBuilders')
- self._filter_mock = self._filter_patcher.start()
- self._filter_mock.side_effect = lambda b, _: b
- self.addCleanup(self._filter_patcher.stop)
- def testErrorOnMixedBuilders(self) -> None:
- """Tests that providing builders of mixed type is an error."""
- builders_to_fill = [
- data_types.BuilderEntry('ci_builder', constants.BuilderTypes.CI, False),
- data_types.BuilderEntry('try_builder', constants.BuilderTypes.TRY,
- False)
- ]
- with self.assertRaises(AssertionError):
- self._querier.FillExpectationMapForBuilders(
- data_types.TestExpectationMap({}), builders_to_fill)
- def testValidResults(self) -> None:
- """Tests functionality when valid results are returned by the query."""
- def SideEffect(builder: data_types.BuilderEntry,
- *args) -> Tuple[data_types.ResultListType, None]:
- del args
- if builder.name == 'matched_builder':
- return ([
- data_types.Result('foo', ['win'], 'Pass', 'step_name', 'build_id')
- ], None)
- if builder.name == 'matched_internal':
- return ([
- data_types.Result('foo', ['win'], 'Pass', 'step_name_internal',
- 'build_id')
- ], None)
- if builder.name == 'unmatched_internal':
- return ([
- data_types.Result('bar', [], 'Pass', 'step_name_internal',
- 'build_id')
- ], None)
- return ([data_types.Result('bar', [], 'Pass', 'step_name',
- 'build_id')], None)
- self._query_mock.side_effect = SideEffect
- expectation = data_types.Expectation('foo', ['win'], 'RetryOnFailure')
- expectation_map = data_types.TestExpectationMap({
- 'foo':
- data_types.ExpectationBuilderMap({
- expectation:
- data_types.BuilderStepMap(),
- }),
- })
- builders_to_fill = [
- data_types.BuilderEntry('matched_builder', constants.BuilderTypes.CI,
- False),
- data_types.BuilderEntry('unmatched_builder', constants.BuilderTypes.CI,
- False),
- data_types.BuilderEntry('matched_internal', constants.BuilderTypes.CI,
- True),
- data_types.BuilderEntry('unmatched_internal', constants.BuilderTypes.CI,
- True),
- ]
- unmatched_results = self._querier.FillExpectationMapForBuilders(
- expectation_map, builders_to_fill)
- stats = data_types.BuildStats()
- stats.AddPassedBuild()
- expected_expectation_map = {
- 'foo': {
- expectation: {
- 'chromium/ci:matched_builder': {
- 'step_name': stats,
- },
- 'chrome/ci:matched_internal': {
- 'step_name_internal': stats,
- },
- },
- },
- }
- self.assertEqual(expectation_map, expected_expectation_map)
- self.assertEqual(
- unmatched_results, {
- 'chromium/ci:unmatched_builder': [
- data_types.Result('bar', [], 'Pass', 'step_name', 'build_id'),
- ],
- 'chrome/ci:unmatched_internal': [
- data_types.Result('bar', [], 'Pass', 'step_name_internal',
- 'build_id'),
- ],
- })
- def testQueryFailureIsSurfaced(self) -> None:
- """Tests that a query failure is properly surfaced despite being async."""
- self._query_mock.side_effect = IndexError('failure')
- with self.assertRaises(IndexError):
- self._querier.FillExpectationMapForBuilders(
- data_types.TestExpectationMap(), [
- data_types.BuilderEntry('matched_builder',
- constants.BuilderTypes.CI, False)
- ])
- class FilterOutInactiveBuildersUnittest(unittest.TestCase):
- def setUp(self) -> None:
- self._subprocess_patcher = mock.patch(
- 'unexpected_passes_common.queries.subprocess.Popen')
- self._subprocess_mock = self._subprocess_patcher.start()
- self.addCleanup(self._subprocess_patcher.stop)
- self._querier = unittest_utils.CreateGenericQuerier()
- def testAllActiveBuilders(self) -> None:
- """Tests that no builders are removed if no inactive builders are found."""
- results = [{
- 'builder_name': 'foo_builder',
- }, {
- 'builder_name': 'bar_builder',
- }]
- fake_process = unittest_utils.FakeProcess(stdout=json.dumps(results))
- self._subprocess_mock.return_value = fake_process
- initial_builders = [
- data_types.BuilderEntry('foo_builder', constants.BuilderTypes.CI,
- False),
- data_types.BuilderEntry('bar_builder', constants.BuilderTypes.CI,
- False),
- ]
- expected_builders = copy.copy(initial_builders)
- filtered_builders = self._querier._FilterOutInactiveBuilders(
- initial_builders, constants.BuilderTypes.CI)
- self.assertEqual(filtered_builders, expected_builders)
- def testInactiveBuilders(self) -> None:
- """Tests that inactive builders are removed."""
- results = [{
- 'builder_name': 'foo_builder',
- }]
- fake_process = unittest_utils.FakeProcess(stdout=json.dumps(results))
- self._subprocess_mock.return_value = fake_process
- initial_builders = [
- data_types.BuilderEntry('foo_builder', constants.BuilderTypes.CI,
- False),
- data_types.BuilderEntry('bar_builder', constants.BuilderTypes.CI,
- False),
- ]
- expected_builders = [
- data_types.BuilderEntry('foo_builder', constants.BuilderTypes.CI, False)
- ]
- filtered_builders = self._querier._FilterOutInactiveBuilders(
- initial_builders, constants.BuilderTypes.CI)
- self.assertEqual(filtered_builders, expected_builders)
- def testByteConversion(self) -> None:
- """Tests that bytes are properly handled if returned."""
- results = [{
- 'builder_name': 'foo_builder',
- }]
- fake_process = unittest_utils.FakeProcess(stdout=json.dumps(results))
- self._subprocess_mock.return_value = fake_process
- initial_builders = [
- data_types.BuilderEntry('foo_builder', constants.BuilderTypes.CI,
- False),
- data_types.BuilderEntry('bar_builder', constants.BuilderTypes.CI,
- False),
- ]
- expected_builders = [
- data_types.BuilderEntry('foo_builder', constants.BuilderTypes.CI, False)
- ]
- filtered_builders = self._querier._FilterOutInactiveBuilders(
- initial_builders, constants.BuilderTypes.CI)
- self.assertEqual(filtered_builders, expected_builders)
- class RunBigQueryCommandsForJsonOutputUnittest(unittest.TestCase):
- def setUp(self) -> None:
- self._popen_patcher = mock.patch.object(subprocess, 'Popen')
- self._popen_mock = self._popen_patcher.start()
- self.addCleanup(self._popen_patcher.stop)
- self._querier = unittest_utils.CreateGenericQuerier()
- def testJsonReturned(self) -> None:
- """Tests that valid JSON parsed from stdout is returned."""
- query_output = [{'foo': 'bar'}]
- self._popen_mock.return_value = unittest_utils.FakeProcess(
- stdout=json.dumps(query_output))
- result = self._querier._RunBigQueryCommandsForJsonOutput([''], {})
- self.assertEqual(result, query_output)
- self._popen_mock.assert_called_once()
- def testJsonReturnedSplitQuery(self) -> None:
- """Tests that valid JSON is returned when a split query is used."""
- def SideEffect(*_, **__) -> unittest_utils.FakeProcess:
- SideEffect.call_count += 1
- if SideEffect.call_count == 1:
- return unittest_utils.FakeProcess(stdout=json.dumps([{'foo': 'bar'}]))
- return unittest_utils.FakeProcess(stdout=json.dumps([{'bar': 'baz'}]))
- SideEffect.call_count = 0
- self._popen_mock.side_effect = SideEffect
- result = self._querier._RunBigQueryCommandsForJsonOutput(['1', '2'], {})
- self.assertEqual(len(result), 2)
- self.assertIn({'foo': 'bar'}, result)
- self.assertIn({'bar': 'baz'}, result)
- def testExceptionRaisedOnFailure(self) -> None:
- """Tests that an exception is raised if the query fails."""
- self._popen_mock.return_value = unittest_utils.FakeProcess(returncode=1)
- with self.assertRaises(RuntimeError):
- self._querier._RunBigQueryCommandsForJsonOutput([''], {})
- def testRateLimitRetrySuccess(self) -> None:
- """Tests that rate limit errors result in retries."""
- def SideEffect(*_, **__) -> unittest_utils.FakeProcess:
- SideEffect.call_count += 1
- if SideEffect.call_count == 1:
- return unittest_utils.FakeProcess(
- returncode=1, stdout='Exceeded rate limits for foo.')
- return unittest_utils.FakeProcess(stdout='[]')
- SideEffect.call_count = 0
- self._popen_mock.side_effect = SideEffect
- self._querier._RunBigQueryCommandsForJsonOutput([''], {})
- self.assertEqual(self._popen_mock.call_count, 2)
- def testRateLimitRetryFailure(self) -> None:
- """Tests that rate limit errors stop retrying after enough iterations."""
- self._popen_mock.return_value = unittest_utils.FakeProcess(
- returncode=1, stdout='Exceeded rate limits for foo.')
- with self.assertRaises(RuntimeError):
- self._querier._RunBigQueryCommandsForJsonOutput([''], {})
- self.assertEqual(self._popen_mock.call_count, queries.MAX_QUERY_TRIES)
- class GenerateBigQueryCommandUnittest(unittest.TestCase):
- def testNoParametersSpecified(self) -> None:
- """Tests that no parameters are added if none are specified."""
- cmd = queries.GenerateBigQueryCommand('project', {})
- for element in cmd:
- self.assertFalse(element.startswith('--parameter'))
- def testParameterAddition(self) -> None:
- """Tests that specified parameters are added appropriately."""
- cmd = queries.GenerateBigQueryCommand('project', {
- '': {
- 'string': 'string_value'
- },
- 'INT64': {
- 'int': 1
- }
- })
- self.assertIn('--parameter=string::string_value', cmd)
- self.assertIn('--parameter=int:INT64:1', cmd)
- def testBatchMode(self) -> None:
- """Tests that batch mode adds the necessary arg."""
- cmd = queries.GenerateBigQueryCommand('project', {}, batch=True)
- self.assertIn('--batch', cmd)
- if __name__ == '__main__':
- unittest.main(verbosity=2)
|