123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616 |
- #!/usr/bin/env vpython3
- # Copyright 2014 The Chromium Authors. All rights reserved.
- # Use of this source code is governed by a BSD-style license that can be
- # found in the LICENSE file.
- import json
- import logging
- import os
- import shutil
- import six
- import sys
- import tempfile
- import unittest
- import common_merge_script_tests
- import six
- THIS_DIR = os.path.dirname(os.path.abspath(__file__))
- # For 'standard_gtest_merge.py'.
- sys.path.insert(
- 0, os.path.abspath(os.path.join(THIS_DIR, '..', 'resources')))
- import mock
- import standard_gtest_merge
- # gtest json output for successfully finished shard #0.
- GOOD_GTEST_JSON_0 = {
- 'all_tests': [
- 'AlignedMemoryTest.DynamicAllocation',
- 'AlignedMemoryTest.ScopedDynamicAllocation',
- 'AlignedMemoryTest.StackAlignment',
- 'AlignedMemoryTest.StaticAlignment',
- ],
- 'disabled_tests': [
- 'ConditionVariableTest.TimeoutAcrossSetTimeOfDay',
- 'FileTest.TouchGetInfo',
- 'MessageLoopTestTypeDefault.EnsureDeletion',
- ],
- 'global_tags': ['CPU_64_BITS', 'MODE_DEBUG', 'OS_LINUX', 'OS_POSIX'],
- 'per_iteration_data': [{
- 'AlignedMemoryTest.DynamicAllocation': [{
- 'elapsed_time_ms': 0,
- 'losless_snippet': True,
- 'output_snippet': 'blah\\n',
- 'output_snippet_base64': 'YmxhaAo=',
- 'status': 'SUCCESS',
- }],
- 'AlignedMemoryTest.ScopedDynamicAllocation': [{
- 'elapsed_time_ms': 0,
- 'losless_snippet': True,
- 'output_snippet': 'blah\\n',
- 'output_snippet_base64': 'YmxhaAo=',
- 'status': 'SUCCESS',
- }],
- }],
- 'test_locations': {
- 'AlignedMemoryTest.DynamicAllocation': {
- 'file': 'foo/bar/allocation_test.cc',
- 'line': 123,
- },
- 'AlignedMemoryTest.ScopedDynamicAllocation': {
- 'file': 'foo/bar/allocation_test.cc',
- 'line': 456,
- },
- # This is a test from a different shard, but this happens in practice and we
- # should not fail if information is repeated.
- 'AlignedMemoryTest.StaticAlignment': {
- 'file': 'foo/bar/allocation_test.cc',
- 'line': 12,
- },
- },
- }
- # gtest json output for successfully finished shard #1.
- GOOD_GTEST_JSON_1 = {
- 'all_tests': [
- 'AlignedMemoryTest.DynamicAllocation',
- 'AlignedMemoryTest.ScopedDynamicAllocation',
- 'AlignedMemoryTest.StackAlignment',
- 'AlignedMemoryTest.StaticAlignment',
- ],
- 'disabled_tests': [
- 'ConditionVariableTest.TimeoutAcrossSetTimeOfDay',
- 'FileTest.TouchGetInfo',
- 'MessageLoopTestTypeDefault.EnsureDeletion',
- ],
- 'global_tags': ['CPU_64_BITS', 'MODE_DEBUG', 'OS_LINUX', 'OS_POSIX'],
- 'per_iteration_data': [{
- 'AlignedMemoryTest.StackAlignment': [{
- 'elapsed_time_ms': 0,
- 'losless_snippet': True,
- 'output_snippet': 'blah\\n',
- 'output_snippet_base64': 'YmxhaAo=',
- 'status': 'SUCCESS',
- }],
- 'AlignedMemoryTest.StaticAlignment': [{
- 'elapsed_time_ms': 0,
- 'losless_snippet': True,
- 'output_snippet': 'blah\\n',
- 'output_snippet_base64': 'YmxhaAo=',
- 'status': 'SUCCESS',
- }],
- }],
- 'test_locations': {
- 'AlignedMemoryTest.StackAlignment': {
- 'file': 'foo/bar/allocation_test.cc',
- 'line': 789,
- },
- 'AlignedMemoryTest.StaticAlignment': {
- 'file': 'foo/bar/allocation_test.cc',
- 'line': 12,
- },
- },
- }
- TIMED_OUT_GTEST_JSON_1 = {
- 'disabled_tests': [],
- 'global_tags': [],
- 'all_tests': [
- 'AlignedMemoryTest.DynamicAllocation',
- 'AlignedMemoryTest.ScopedDynamicAllocation',
- 'AlignedMemoryTest.StackAlignment',
- 'AlignedMemoryTest.StaticAlignment',
- ],
- 'per_iteration_data': [{
- 'AlignedMemoryTest.StackAlignment': [{
- 'elapsed_time_ms': 54000,
- 'losless_snippet': True,
- 'output_snippet': 'timed out',
- 'output_snippet_base64': '',
- 'status': 'FAILURE',
- }],
- 'AlignedMemoryTest.StaticAlignment': [{
- 'elapsed_time_ms': 0,
- 'losless_snippet': True,
- 'output_snippet': '',
- 'output_snippet_base64': '',
- 'status': 'NOTRUN',
- }],
- }],
- 'test_locations': {
- 'AlignedMemoryTest.StackAlignment': {
- 'file': 'foo/bar/allocation_test.cc',
- 'line': 789,
- },
- 'AlignedMemoryTest.StaticAlignment': {
- 'file': 'foo/bar/allocation_test.cc',
- 'line': 12,
- },
- },
- }
- # GOOD_GTEST_JSON_0 and GOOD_GTEST_JSON_1 merged.
- GOOD_GTEST_JSON_MERGED = {
- 'all_tests': [
- 'AlignedMemoryTest.DynamicAllocation',
- 'AlignedMemoryTest.ScopedDynamicAllocation',
- 'AlignedMemoryTest.StackAlignment',
- 'AlignedMemoryTest.StaticAlignment',
- ],
- 'disabled_tests': [
- 'ConditionVariableTest.TimeoutAcrossSetTimeOfDay',
- 'FileTest.TouchGetInfo',
- 'MessageLoopTestTypeDefault.EnsureDeletion',
- ],
- 'global_tags': ['CPU_64_BITS', 'MODE_DEBUG', 'OS_LINUX', 'OS_POSIX'],
- 'missing_shards': [],
- 'per_iteration_data': [{
- 'AlignedMemoryTest.DynamicAllocation': [{
- 'elapsed_time_ms': 0,
- 'losless_snippet': True,
- 'output_snippet': 'blah\\n',
- 'output_snippet_base64': 'YmxhaAo=',
- 'status': 'SUCCESS',
- }],
- 'AlignedMemoryTest.ScopedDynamicAllocation': [{
- 'elapsed_time_ms': 0,
- 'losless_snippet': True,
- 'output_snippet': 'blah\\n',
- 'output_snippet_base64': 'YmxhaAo=',
- 'status': 'SUCCESS',
- }],
- 'AlignedMemoryTest.StackAlignment': [{
- 'elapsed_time_ms': 0,
- 'losless_snippet': True,
- 'output_snippet': 'blah\\n',
- 'output_snippet_base64': 'YmxhaAo=',
- 'status': 'SUCCESS',
- }],
- 'AlignedMemoryTest.StaticAlignment': [{
- 'elapsed_time_ms': 0,
- 'losless_snippet': True,
- 'output_snippet': 'blah\\n',
- 'output_snippet_base64': 'YmxhaAo=',
- 'status': 'SUCCESS',
- }],
- }],
- 'swarming_summary': {
- u'shards': [
- {
- u'state': u'COMPLETED',
- u'outputs_ref': {
- u'view_url': u'blah',
- },
- }
- ],
- },
- 'test_locations': {
- 'AlignedMemoryTest.StackAlignment': {
- 'file': 'foo/bar/allocation_test.cc',
- 'line': 789,
- },
- 'AlignedMemoryTest.StaticAlignment': {
- 'file': 'foo/bar/allocation_test.cc',
- 'line': 12,
- },
- 'AlignedMemoryTest.DynamicAllocation': {
- 'file': 'foo/bar/allocation_test.cc',
- 'line': 123,
- },
- 'AlignedMemoryTest.ScopedDynamicAllocation': {
- 'file': 'foo/bar/allocation_test.cc',
- 'line': 456,
- },
- },
- }
- # Only shard #1 finished. UNRELIABLE_RESULTS is set.
- BAD_GTEST_JSON_ONLY_1_SHARD = {
- 'all_tests': [
- 'AlignedMemoryTest.DynamicAllocation',
- 'AlignedMemoryTest.ScopedDynamicAllocation',
- 'AlignedMemoryTest.StackAlignment',
- 'AlignedMemoryTest.StaticAlignment',
- ],
- 'disabled_tests': [
- 'ConditionVariableTest.TimeoutAcrossSetTimeOfDay',
- 'FileTest.TouchGetInfo',
- 'MessageLoopTestTypeDefault.EnsureDeletion',
- ],
- 'global_tags': [
- 'CPU_64_BITS',
- 'MODE_DEBUG',
- 'OS_LINUX',
- 'OS_POSIX',
- 'UNRELIABLE_RESULTS',
- ],
- 'missing_shards': [0],
- 'per_iteration_data': [{
- 'AlignedMemoryTest.StackAlignment': [{
- 'elapsed_time_ms': 0,
- 'losless_snippet': True,
- 'output_snippet': 'blah\\n',
- 'output_snippet_base64': 'YmxhaAo=',
- 'status': 'SUCCESS',
- }],
- 'AlignedMemoryTest.StaticAlignment': [{
- 'elapsed_time_ms': 0,
- 'losless_snippet': True,
- 'output_snippet': 'blah\\n',
- 'output_snippet_base64': 'YmxhaAo=',
- 'status': 'SUCCESS',
- }],
- }],
- 'test_locations': {
- 'AlignedMemoryTest.StackAlignment': {
- 'file': 'foo/bar/allocation_test.cc',
- 'line': 789,
- },
- 'AlignedMemoryTest.StaticAlignment': {
- 'file': 'foo/bar/allocation_test.cc',
- 'line': 12,
- },
- },
- }
- # GOOD_GTEST_JSON_0 and TIMED_OUT_GTEST_JSON_1 merged.
- TIMED_OUT_GTEST_JSON_MERGED = {
- 'all_tests': [
- 'AlignedMemoryTest.DynamicAllocation',
- 'AlignedMemoryTest.ScopedDynamicAllocation',
- 'AlignedMemoryTest.StackAlignment',
- 'AlignedMemoryTest.StaticAlignment',
- ],
- 'disabled_tests': [
- 'ConditionVariableTest.TimeoutAcrossSetTimeOfDay',
- 'FileTest.TouchGetInfo',
- 'MessageLoopTestTypeDefault.EnsureDeletion',
- ],
- 'global_tags': ['CPU_64_BITS', 'MODE_DEBUG', 'OS_LINUX', 'OS_POSIX'],
- 'missing_shards': [],
- 'per_iteration_data': [{
- 'AlignedMemoryTest.DynamicAllocation': [{
- 'elapsed_time_ms': 0,
- 'losless_snippet': True,
- 'output_snippet': 'blah\\n',
- 'output_snippet_base64': 'YmxhaAo=',
- 'status': 'SUCCESS',
- }],
- 'AlignedMemoryTest.ScopedDynamicAllocation': [{
- 'elapsed_time_ms': 0,
- 'losless_snippet': True,
- 'output_snippet': 'blah\\n',
- 'output_snippet_base64': 'YmxhaAo=',
- 'status': 'SUCCESS',
- }],
- 'AlignedMemoryTest.StackAlignment': [{
- 'elapsed_time_ms': 54000,
- 'losless_snippet': True,
- 'output_snippet': 'timed out',
- 'output_snippet_base64': '',
- 'status': 'FAILURE',
- }],
- 'AlignedMemoryTest.StaticAlignment': [{
- 'elapsed_time_ms': 0,
- 'losless_snippet': True,
- 'output_snippet': '',
- 'output_snippet_base64': '',
- 'status': 'NOTRUN',
- }],
- }],
- 'swarming_summary': {
- u'shards': [
- {
- u'state': u'COMPLETED',
- },
- {
- u'state': u'TIMED_OUT',
- },
- ],
- },
- 'test_locations': {
- 'AlignedMemoryTest.StackAlignment': {
- 'file': 'foo/bar/allocation_test.cc',
- 'line': 789,
- },
- 'AlignedMemoryTest.StaticAlignment': {
- 'file': 'foo/bar/allocation_test.cc',
- 'line': 12,
- },
- 'AlignedMemoryTest.DynamicAllocation': {
- 'file': 'foo/bar/allocation_test.cc',
- 'line': 123,
- },
- 'AlignedMemoryTest.ScopedDynamicAllocation': {
- 'file': 'foo/bar/allocation_test.cc',
- 'line': 456,
- },
- },
- }
- class _StandardGtestMergeTest(unittest.TestCase):
- def setUp(self):
- self.temp_dir = tempfile.mkdtemp()
- def tearDown(self):
- shutil.rmtree(self.temp_dir)
- def _write_temp_file(self, path, content):
- abs_path = os.path.join(self.temp_dir, path.replace('/', os.sep))
- if not os.path.exists(os.path.dirname(abs_path)):
- os.makedirs(os.path.dirname(abs_path))
- with open(abs_path, 'w') as f:
- if isinstance(content, dict):
- json.dump(content, f)
- else:
- assert isinstance(content, str)
- f.write(content)
- return abs_path
- class LoadShardJsonTest(_StandardGtestMergeTest):
- def test_double_digit_jsons(self):
- jsons_to_merge = []
- for i in range(15):
- json_dir = os.path.join(self.temp_dir, str(i))
- json_path = os.path.join(json_dir, 'output.json')
- if not os.path.exists(json_dir):
- os.makedirs(json_dir)
- with open(json_path, 'w') as f:
- json.dump({'all_tests': ['LoadShardJsonTest.test%d' % i]}, f)
- jsons_to_merge.append(json_path)
- content, err = standard_gtest_merge.load_shard_json(
- 0, None, jsons_to_merge)
- self.assertEqual({'all_tests': ['LoadShardJsonTest.test0']}, content)
- self.assertIsNone(err)
- content, err = standard_gtest_merge.load_shard_json(
- 12, None, jsons_to_merge)
- self.assertEqual({'all_tests': ['LoadShardJsonTest.test12']}, content)
- self.assertIsNone(err)
- def test_double_task_id_jsons(self):
- jsons_to_merge = []
- for i in range(15):
- json_dir = os.path.join(self.temp_dir, 'deadbeef%d' % i)
- json_path = os.path.join(json_dir, 'output.json')
- if not os.path.exists(json_dir):
- os.makedirs(json_dir)
- with open(json_path, 'w') as f:
- json.dump({'all_tests': ['LoadShardJsonTest.test%d' % i]}, f)
- jsons_to_merge.append(json_path)
- content, err = standard_gtest_merge.load_shard_json(
- 0, 'deadbeef0', jsons_to_merge)
- self.assertEqual({'all_tests': ['LoadShardJsonTest.test0']},
- content)
- self.assertIsNone(err)
- content, err = standard_gtest_merge.load_shard_json(
- 12, 'deadbeef12', jsons_to_merge)
- self.assertEqual({'all_tests': ['LoadShardJsonTest.test12']},
- content)
- self.assertIsNone(err)
- class MergeShardResultsTest(_StandardGtestMergeTest):
- """Tests for merge_shard_results function."""
- # pylint: disable=super-with-arguments
- def setUp(self):
- super(MergeShardResultsTest, self).setUp()
- self.summary = None
- self.test_files = []
- # pylint: enable=super-with-arguments
- def stage(self, summary, files):
- self.summary = self._write_temp_file('summary.json', summary)
- for path, content in files.items():
- abs_path = self._write_temp_file(path, content)
- self.test_files.append(abs_path)
- def call(self):
- stdout = six.StringIO()
- with mock.patch('sys.stdout', stdout):
- merged = standard_gtest_merge.merge_shard_results(
- self.summary, self.test_files)
- return merged, stdout.getvalue().strip()
- def assertUnicodeEquals(self, expectation, result):
- def convert_to_unicode(key_or_value):
- if isinstance(key_or_value, str):
- return six.text_type(key_or_value)
- if isinstance(key_or_value, dict):
- return {convert_to_unicode(k): convert_to_unicode(v)
- for k, v in key_or_value.items()}
- if isinstance(key_or_value, list):
- return [convert_to_unicode(x) for x in key_or_value]
- return key_or_value
- unicode_expectations = convert_to_unicode(expectation)
- unicode_result = convert_to_unicode(result)
- self.assertEquals(unicode_expectations, unicode_result)
- def test_ok(self):
- # Two shards, both successfully finished.
- self.stage({
- u'shards': [
- {
- u'state': u'COMPLETED',
- },
- {
- u'state': u'COMPLETED',
- },
- ],
- },
- {
- '0/output.json': GOOD_GTEST_JSON_0,
- '1/output.json': GOOD_GTEST_JSON_1,
- })
- merged, stdout = self.call()
- merged['swarming_summary'] = {
- 'shards': [
- {
- u'state': u'COMPLETED',
- u'outputs_ref': {
- u'view_url': u'blah',
- },
- }
- ],
- }
- self.assertUnicodeEquals(GOOD_GTEST_JSON_MERGED, merged)
- self.assertEqual('', stdout)
- def test_timed_out(self):
- # Two shards, both successfully finished.
- self.stage({
- 'shards': [
- {
- 'state': 'COMPLETED',
- },
- {
- 'state': 'TIMED_OUT',
- },
- ],
- },
- {
- '0/output.json': GOOD_GTEST_JSON_0,
- '1/output.json': TIMED_OUT_GTEST_JSON_1,
- })
- merged, stdout = self.call()
- self.assertUnicodeEquals(TIMED_OUT_GTEST_JSON_MERGED, merged)
- self.assertIn(
- 'Test runtime exceeded allocated time\n', stdout)
- def test_missing_summary_json(self):
- # summary.json is missing, should return None and emit warning.
- self.summary = os.path.join(self.temp_dir, 'summary.json')
- merged, output = self.call()
- self.assertEqual(None, merged)
- self.assertIn('@@@STEP_WARNINGS@@@', output)
- self.assertIn('summary.json is missing or can not be read', output)
- def test_unfinished_shards(self):
- # Only one shard (#1) finished. Shard #0 did not.
- self.stage({
- u'shards': [
- None,
- {
- u'state': u'COMPLETED',
- },
- ],
- },
- {
- u'1/output.json': GOOD_GTEST_JSON_1,
- })
- merged, stdout = self.call()
- merged.pop('swarming_summary')
- self.assertUnicodeEquals(BAD_GTEST_JSON_ONLY_1_SHARD, merged)
- self.assertIn(
- '@@@STEP_WARNINGS@@@\nsome shards did not complete: 0\n', stdout)
- self.assertIn(
- '@@@STEP_LOG_LINE@some shards did not complete: 0@'
- 'Missing results from the following shard(s): 0@@@\n', stdout)
- def test_missing_output_json(self):
- # Shard #0 output json is missing.
- self.stage({
- u'shards': [
- {
- u'state': u'COMPLETED',
- },
- {
- u'state': u'COMPLETED',
- },
- ],
- },
- {
- u'1/output.json': GOOD_GTEST_JSON_1,
- })
- merged, stdout = self.call()
- merged.pop('swarming_summary')
- self.assertUnicodeEquals(BAD_GTEST_JSON_ONLY_1_SHARD, merged)
- self.assertIn(
- 'No result was found: '
- 'shard 0 test output was missing', stdout)
- def test_large_output_json(self):
- # a shard is too large.
- self.stage({
- u'shards': [
- {
- u'state': u'COMPLETED',
- },
- {
- u'state': u'COMPLETED',
- },
- ],
- },
- {
- '0/output.json': GOOD_GTEST_JSON_0,
- '1/output.json': GOOD_GTEST_JSON_1,
- })
- old_json_limit = standard_gtest_merge.OUTPUT_JSON_SIZE_LIMIT
- len0 = len(json.dumps(GOOD_GTEST_JSON_0))
- len1 = len(json.dumps(GOOD_GTEST_JSON_1))
- large_shard = "0" if len0 > len1 else "1"
- try:
- # Override max output.json size just for this test.
- standard_gtest_merge.OUTPUT_JSON_SIZE_LIMIT = min(len0,len1)
- merged, stdout = self.call()
- merged.pop('swarming_summary')
- self.assertUnicodeEquals(BAD_GTEST_JSON_ONLY_1_SHARD, merged)
- self.assertIn(
- 'No result was found: '
- 'shard %s test output exceeded the size limit' % large_shard, stdout)
- finally:
- standard_gtest_merge.OUTPUT_JSON_SIZE_LIMIT = old_json_limit
- class CommandLineTest(common_merge_script_tests.CommandLineTest):
- # pylint: disable=super-with-arguments
- def __init__(self, methodName='runTest'):
- super(CommandLineTest, self).__init__(methodName, standard_gtest_merge)
- # pylint: enable=super-with-arguments
- if __name__ == '__main__':
- logging.basicConfig(
- level=logging.DEBUG if '-v' in sys.argv else logging.ERROR)
- if '-v' in sys.argv:
- unittest.TestCase.maxDiff = None
- unittest.main()
|