results_merger.py 8.5 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284
  1. #!/usr/bin/env python
  2. # Copyright 2016 The Chromium Authors. All rights reserved.
  3. # Use of this source code is governed by a BSD-style license that can be
  4. # found in the LICENSE file.
  5. from __future__ import print_function
  6. import copy
  7. import json
  8. import sys
  9. # These fields must appear in the test result output
  10. REQUIRED = {
  11. 'interrupted',
  12. 'num_failures_by_type',
  13. 'seconds_since_epoch',
  14. 'tests',
  15. }
  16. # These fields are optional, but must have the same value on all shards
  17. OPTIONAL_MATCHING = (
  18. 'builder_name',
  19. 'build_number',
  20. 'chromium_revision',
  21. 'has_pretty_patch',
  22. 'has_wdiff',
  23. 'path_delimiter',
  24. 'pixel_tests_enabled',
  25. 'random_order_seed'
  26. )
  27. # The last shard's value for these fields will show up in the merged results
  28. OPTIONAL_IGNORED = (
  29. 'layout_tests_dir',
  30. 'metadata'
  31. )
  32. # These fields are optional and will be summed together
  33. OPTIONAL_COUNTS = (
  34. 'fixable',
  35. 'num_flaky',
  36. 'num_passes',
  37. 'num_regressions',
  38. 'skipped',
  39. 'skips',
  40. )
  41. class MergeException(Exception):
  42. pass
  43. def merge_test_results(shard_results_list):
  44. """ Merge list of results.
  45. Args:
  46. shard_results_list: list of results to merge. All the results must have the
  47. same format. Supported format are simplified JSON format & Chromium JSON
  48. test results format version 3 (see
  49. https://www.chromium.org/developers/the-json-test-results-format)
  50. Returns:
  51. a dictionary that represent the merged results. Its format follow the same
  52. format of all results in |shard_results_list|.
  53. """
  54. shard_results_list = [x for x in shard_results_list if x]
  55. if not shard_results_list:
  56. return {}
  57. if 'seconds_since_epoch' in shard_results_list[0]:
  58. return _merge_json_test_result_format(shard_results_list)
  59. return _merge_simplified_json_format(shard_results_list)
  60. def _merge_simplified_json_format(shard_results_list):
  61. # This code is specialized to the "simplified" JSON format that used to be
  62. # the standard for recipes.
  63. # These are the only keys we pay attention to in the output JSON.
  64. merged_results = {
  65. 'successes': [],
  66. 'failures': [],
  67. 'valid': True,
  68. }
  69. for result_json in shard_results_list:
  70. successes = result_json.get('successes', [])
  71. failures = result_json.get('failures', [])
  72. valid = result_json.get('valid', True)
  73. if (not isinstance(successes, list) or not isinstance(failures, list) or
  74. not isinstance(valid, bool)):
  75. raise MergeException(
  76. 'Unexpected value type in %s' % result_json) # pragma: no cover
  77. merged_results['successes'].extend(successes)
  78. merged_results['failures'].extend(failures)
  79. merged_results['valid'] = merged_results['valid'] and valid
  80. return merged_results
  81. def _merge_json_test_result_format(shard_results_list):
  82. # This code is specialized to the Chromium JSON test results format version 3:
  83. # https://www.chromium.org/developers/the-json-test-results-format
  84. # These are required fields for the JSON test result format version 3.
  85. merged_results = {
  86. 'tests': {},
  87. 'interrupted': False,
  88. 'version': 3,
  89. 'seconds_since_epoch': float('inf'),
  90. 'num_failures_by_type': {
  91. }
  92. }
  93. # To make sure that we don't mutate existing shard_results_list.
  94. shard_results_list = copy.deepcopy(shard_results_list)
  95. for result_json in shard_results_list:
  96. # TODO(tansell): check whether this deepcopy is actually necessary.
  97. result_json = copy.deepcopy(result_json)
  98. # Check the version first
  99. version = result_json.pop('version', -1)
  100. if version != 3:
  101. raise MergeException( # pragma: no cover (covered by
  102. # results_merger_unittest).
  103. 'Unsupported version %s. Only version 3 is supported' % version)
  104. # Check the results for each shard have the required keys
  105. missing = REQUIRED - set(result_json)
  106. if missing:
  107. raise MergeException( # pragma: no cover (covered by
  108. # results_merger_unittest).
  109. 'Invalid json test results (missing %s)' % missing)
  110. # Curry merge_values for this result_json.
  111. merge = lambda key, merge_func: merge_value(
  112. result_json, merged_results, key, merge_func)
  113. # Traverse the result_json's test trie & merged_results's test tries in
  114. # DFS order & add the n to merged['tests'].
  115. merge('tests', merge_tries)
  116. # If any were interrupted, we are interrupted.
  117. merge('interrupted', lambda x,y: x|y)
  118. # Use the earliest seconds_since_epoch value
  119. merge('seconds_since_epoch', min)
  120. # Sum the number of failure types
  121. merge('num_failures_by_type', sum_dicts)
  122. # Optional values must match
  123. for optional_key in OPTIONAL_MATCHING:
  124. if optional_key not in result_json:
  125. continue
  126. if optional_key not in merged_results:
  127. # Set this value to None, then blindly copy over it.
  128. merged_results[optional_key] = None
  129. merge(optional_key, lambda src, dst: src)
  130. else:
  131. merge(optional_key, ensure_match)
  132. # Optional values ignored
  133. for optional_key in OPTIONAL_IGNORED:
  134. if optional_key in result_json:
  135. merged_results[optional_key] = result_json.pop(
  136. # pragma: no cover (covered by
  137. # results_merger_unittest).
  138. optional_key)
  139. # Sum optional value counts
  140. for count_key in OPTIONAL_COUNTS:
  141. if count_key in result_json: # pragma: no cover
  142. # TODO(mcgreevy): add coverage.
  143. merged_results.setdefault(count_key, 0)
  144. merge(count_key, lambda a, b: a+b)
  145. if result_json:
  146. raise MergeException( # pragma: no cover (covered by
  147. # results_merger_unittest).
  148. 'Unmergable values %s' % list(result_json.keys()))
  149. return merged_results
  150. def merge_tries(source, dest):
  151. """ Merges test tries.
  152. This is intended for use as a merge_func parameter to merge_value.
  153. Args:
  154. source: A result json test trie.
  155. dest: A json test trie merge destination.
  156. """
  157. # merge_tries merges source into dest by performing a lock-step depth-first
  158. # traversal of dest and source.
  159. # pending_nodes contains a list of all sub-tries which have been reached but
  160. # need further merging.
  161. # Each element consists of a trie prefix, and a sub-trie from each of dest
  162. # and source which is reached via that prefix.
  163. pending_nodes = [('', dest, source)]
  164. while pending_nodes:
  165. prefix, dest_node, curr_node = pending_nodes.pop()
  166. for k, v in curr_node.items():
  167. if k in dest_node:
  168. if not isinstance(v, dict):
  169. raise MergeException(
  170. "%s:%s: %r not mergable, curr_node: %r\ndest_node: %r" % (
  171. prefix, k, v, curr_node, dest_node))
  172. pending_nodes.append(("%s:%s" % (prefix, k), dest_node[k], v))
  173. else:
  174. dest_node[k] = v
  175. return dest
  176. def ensure_match(source, dest):
  177. """ Returns source if it matches dest.
  178. This is intended for use as a merge_func parameter to merge_value.
  179. Raises:
  180. MergeException if source != dest
  181. """
  182. if source != dest:
  183. raise MergeException( # pragma: no cover (covered by
  184. # results_merger_unittest).
  185. "Values don't match: %s, %s" % (source, dest))
  186. return source
  187. def sum_dicts(source, dest):
  188. """ Adds values from source to corresponding values in dest.
  189. This is intended for use as a merge_func parameter to merge_value.
  190. """
  191. for k, v in source.items():
  192. dest.setdefault(k, 0)
  193. dest[k] += v
  194. return dest
  195. def merge_value(source, dest, key, merge_func):
  196. """ Merges a value from source to dest.
  197. The value is deleted from source.
  198. Args:
  199. source: A dictionary from which to pull a value, identified by key.
  200. dest: The dictionary into to which the value is to be merged.
  201. key: The key which identifies the value to be merged.
  202. merge_func(src, dst): A function which merges its src into dst,
  203. and returns the result. May modify dst. May raise a MergeException.
  204. Raises:
  205. MergeException if the values can not be merged.
  206. """
  207. try:
  208. dest[key] = merge_func(source[key], dest[key])
  209. except MergeException as e:
  210. message = "MergeFailure for %s\n%s" % (key, e.args[0])
  211. e.args = (message,) + e.args[1:]
  212. raise
  213. del source[key]
  214. def main(files):
  215. if len(files) < 2:
  216. sys.stderr.write("Not enough JSON files to merge.\n")
  217. return 1
  218. sys.stderr.write('Starting with %s\n' % files[0])
  219. result = json.load(open(files[0]))
  220. for f in files[1:]:
  221. sys.stderr.write('Merging %s\n' % f)
  222. result = merge_test_results([result, json.load(open(f))])
  223. print(json.dumps(result))
  224. return 0
  225. if __name__ == "__main__":
  226. sys.exit(main(sys.argv[1:]))