results_merger_test.py 7.9 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296
  1. #!/usr/bin/env vpython3
  2. # Copyright 2016 The Chromium Authors. All rights reserved.
  3. # Use of this source code is governed by a BSD-style license that can be
  4. # found in the LICENSE file.
  5. import copy
  6. import os
  7. import six
  8. import sys
  9. import unittest
  10. THIS_DIR = os.path.dirname(os.path.abspath(__file__))
  11. # For results_merger.
  12. sys.path.insert(0, os.path.join(THIS_DIR, '..', 'resources'))
  13. import results_merger
  14. GOOD_JSON_TEST_RESULT_0 = {
  15. 'tests': {
  16. 'car': {
  17. 'honda': {
  18. 'expected': 'PASS',
  19. 'actual': 'PASS'
  20. },
  21. 'toyota': {
  22. 'expected': 'FAIL',
  23. 'actual': 'FAIL'
  24. }
  25. },
  26. 'computer': {
  27. 'dell': {
  28. 'expected': 'PASS',
  29. 'actual': 'PASS'
  30. }
  31. },
  32. },
  33. 'interrupted': False,
  34. 'version': 3,
  35. 'seconds_since_epoch': 1406662289.76,
  36. 'num_failures_by_type': {
  37. 'FAIL': 0,
  38. 'PASS': 2
  39. },
  40. 'layout_tests_dir': 'abc'
  41. }
  42. GOOD_JSON_TEST_RESULT_1 = {
  43. 'tests': {
  44. 'car': {
  45. 'tesla': {
  46. 'expected': 'PASS',
  47. 'actual': 'PASS'
  48. },
  49. },
  50. 'burger': {
  51. 'mcdonald': {
  52. 'expected': 'PASS',
  53. 'actual': 'PASS'
  54. }
  55. },
  56. },
  57. 'interrupted': False,
  58. 'version': 3,
  59. 'seconds_since_epoch': 1406662283.11,
  60. 'num_failures_by_type': {
  61. 'FAIL': 0,
  62. 'PASS': 2
  63. },
  64. 'layout_tests_dir': '123'
  65. }
  66. GOOD_JSON_TEST_RESULT_2 = {
  67. 'tests': {
  68. 'car': {
  69. 'mercedes': {
  70. 'expected': 'PASS',
  71. 'actual': 'FAIL'
  72. },
  73. },
  74. 'burger': {
  75. 'in n out': {
  76. 'expected': 'PASS',
  77. 'actual': 'PASS'
  78. }
  79. },
  80. },
  81. 'interrupted': True,
  82. 'version': 3,
  83. 'seconds_since_epoch': 1406662200.01,
  84. 'num_failures_by_type': {
  85. 'FAIL': 1,
  86. 'PASS': 1
  87. }
  88. }
  89. GOOD_JSON_TEST_RESULT_MERGED = {
  90. 'tests': {
  91. 'car': {
  92. 'tesla': {
  93. 'expected': 'PASS',
  94. 'actual': 'PASS'
  95. },
  96. 'mercedes': {
  97. 'expected': 'PASS',
  98. 'actual': 'FAIL'
  99. },
  100. 'honda': {
  101. 'expected': 'PASS',
  102. 'actual': 'PASS'
  103. },
  104. 'toyota': {
  105. 'expected': 'FAIL',
  106. 'actual': 'FAIL'
  107. }
  108. },
  109. 'computer': {
  110. 'dell': {
  111. 'expected': 'PASS',
  112. 'actual': 'PASS'
  113. }
  114. },
  115. 'burger': {
  116. 'mcdonald': {
  117. 'expected': 'PASS',
  118. 'actual': 'PASS'
  119. },
  120. 'in n out': {
  121. 'expected': 'PASS',
  122. 'actual': 'PASS'
  123. }
  124. }
  125. },
  126. 'interrupted': True,
  127. 'version': 3,
  128. 'seconds_since_epoch': 1406662200.01,
  129. 'num_failures_by_type': {
  130. 'FAIL': 1,
  131. 'PASS': 5
  132. },
  133. 'layout_tests_dir': '123'
  134. }
  135. def extend(initial, add):
  136. out = copy.deepcopy(initial)
  137. out.update(add)
  138. return out
  139. def remove(initial, keys):
  140. out = copy.deepcopy(initial)
  141. for k in keys:
  142. del out[k]
  143. return out
  144. # These unittests are run in PRESUBMIT, but not by recipe_simulation_test, hence
  145. # to avoid false alert on missing coverage by recipe_simulation_test, we mark
  146. # these code as no cover.
  147. class MergingTest(unittest.TestCase): # pragma: no cover
  148. maxDiff = None # Show full diff if assertion fail
  149. def test_merge_tries(self):
  150. self.assertEqual(
  151. {'a': 'A', 'b': {'c': 'C'}},
  152. results_merger.merge_tries(
  153. {'a': 'A', 'b': {}}, {'b': {'c': 'C'}}))
  154. def test_merge_tries_unmergable(self):
  155. with six.assertRaisesRegex(self, results_merger.MergeException, "a:b"):
  156. results_merger.merge_tries(
  157. {'a': {'b': 'A'}}, {'a': {'b': 'C'}})
  158. def test_merge_metadata(self):
  159. metadata1 = {'metadata': {'tags': ['foo', 'bar']}}
  160. metadata2 = {'metadata': {'tags': ['foo', 'bat']}}
  161. merged_results = results_merger.merge_test_results(
  162. [extend(GOOD_JSON_TEST_RESULT_0, metadata1),
  163. extend(GOOD_JSON_TEST_RESULT_1, metadata2)])
  164. self.assertEqual(
  165. merged_results['metadata']['tags'], ['foo', 'bat'])
  166. def test_merge_json_test_results_nop(self):
  167. good_json_results = (
  168. GOOD_JSON_TEST_RESULT_0,
  169. GOOD_JSON_TEST_RESULT_1,
  170. GOOD_JSON_TEST_RESULT_2,
  171. GOOD_JSON_TEST_RESULT_MERGED)
  172. for j in good_json_results:
  173. # Clone so we can check the input dictionaries are not modified
  174. a = copy.deepcopy(j)
  175. self.assertEqual(results_merger.merge_test_results([a]), j)
  176. self.assertEqual(a, j)
  177. def test_merge_json_test_results_invalid_version(self):
  178. with self.assertRaises(results_merger.MergeException):
  179. results_merger.merge_test_results([
  180. extend(GOOD_JSON_TEST_RESULT_0, {'version': 5}),
  181. ])
  182. with self.assertRaises(results_merger.MergeException):
  183. results_merger.merge_test_results([
  184. GOOD_JSON_TEST_RESULT_0,
  185. extend(GOOD_JSON_TEST_RESULT_1, {'version': 5}),
  186. ])
  187. def test_merge_json_test_results_missing_version(self):
  188. with self.assertRaises(results_merger.MergeException):
  189. results_merger.merge_test_results([
  190. remove(GOOD_JSON_TEST_RESULT_0, ['version']),
  191. ])
  192. with self.assertRaises(results_merger.MergeException):
  193. results_merger.merge_test_results([
  194. GOOD_JSON_TEST_RESULT_0,
  195. remove(GOOD_JSON_TEST_RESULT_1, ['version']),
  196. ])
  197. def test_merge_json_test_results_invalid_extra(self):
  198. with self.assertRaises(results_merger.MergeException):
  199. results_merger.merge_test_results([
  200. extend(GOOD_JSON_TEST_RESULT_0, {'extra': True}),
  201. ])
  202. with self.assertRaises(results_merger.MergeException):
  203. results_merger.merge_test_results([
  204. GOOD_JSON_TEST_RESULT_0,
  205. extend(GOOD_JSON_TEST_RESULT_1, {'extra': True}),
  206. ])
  207. def test_merge_json_test_results_missing_required(self):
  208. with self.assertRaises(results_merger.MergeException):
  209. results_merger.merge_test_results([
  210. remove(GOOD_JSON_TEST_RESULT_0, ['interrupted']),
  211. ])
  212. with self.assertRaises(results_merger.MergeException):
  213. results_merger.merge_test_results([
  214. GOOD_JSON_TEST_RESULT_0,
  215. remove(GOOD_JSON_TEST_RESULT_1, ['interrupted']),
  216. ])
  217. def test_merge_json_test_results_multiple(self):
  218. self.assertEqual(
  219. results_merger.merge_test_results([
  220. GOOD_JSON_TEST_RESULT_0,
  221. GOOD_JSON_TEST_RESULT_1,
  222. GOOD_JSON_TEST_RESULT_2,
  223. ]),
  224. GOOD_JSON_TEST_RESULT_MERGED)
  225. def test_merge_json_test_results_optional_matches(self):
  226. self.assertEqual(
  227. results_merger.merge_test_results([
  228. extend(GOOD_JSON_TEST_RESULT_0, {'path_delimiter': '.'}),
  229. extend(GOOD_JSON_TEST_RESULT_1, {'path_delimiter': '.'}),
  230. extend(GOOD_JSON_TEST_RESULT_2, {'path_delimiter': '.'}),
  231. ]),
  232. extend(GOOD_JSON_TEST_RESULT_MERGED, {'path_delimiter': '.'}))
  233. def test_merge_json_test_results_optional_differs(self):
  234. with self.assertRaises(results_merger.MergeException):
  235. results_merger.merge_test_results([
  236. extend(GOOD_JSON_TEST_RESULT_0, {'path_delimiter': '.'}),
  237. extend(GOOD_JSON_TEST_RESULT_1, {'path_delimiter': '.'}),
  238. extend(GOOD_JSON_TEST_RESULT_2, {'path_delimiter': '/'}),
  239. ])
  240. def test_merge_json_test_results_optional_count(self):
  241. self.assertEqual(
  242. results_merger.merge_test_results([
  243. extend(GOOD_JSON_TEST_RESULT_0, {'fixable': 1}),
  244. extend(GOOD_JSON_TEST_RESULT_1, {'fixable': 2}),
  245. extend(GOOD_JSON_TEST_RESULT_2, {'fixable': 3}),
  246. ]),
  247. extend(GOOD_JSON_TEST_RESULT_MERGED, {'fixable': 6}))
  248. def test_merge_nothing(self):
  249. self.assertEqual(
  250. results_merger.merge_test_results([]),
  251. {})
  252. # TODO(tansell): Make this test fail properly, currently fails with an
  253. # AttributeError.
  254. # def test_merge_test_name_conflict(self):
  255. # self.maxDiff = None # Show full diff if assertion fail
  256. # with self.assertRaises(results_merger.MergeException):
  257. # results_merger.merge_test_results(
  258. # [GOOD_JSON_TEST_RESULT_0, GOOD_JSON_TEST_RESULT_0])
  259. if __name__ == '__main__':
  260. unittest.main() # pragma: no cover