test_results.py 5.1 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145
  1. # Copyright 2021 The Chromium Authors. All rights reserved.
  2. # Use of this source code is governed by a BSD-style license that can be
  3. # found in the LICENSE file.
  4. """This is a library for printing test result as specified by
  5. //docs/testing/test_executable_api.md (e.g. --isolated-script-test-output)
  6. and //docs/testing/json_test_results_format.md
  7. Typical usage:
  8. import argparse
  9. import test_results
  10. cmdline_parser = argparse.ArgumentParser()
  11. test_results.add_cmdline_args(cmdline_parser)
  12. ... adding other cmdline parameter definitions ...
  13. parsed_cmdline_args = cmdline_parser.parse_args()
  14. test_results = []
  15. test_results.append(test_results.TestResult(
  16. 'test-suite/test-name', 'PASS'))
  17. ...
  18. test_results.print_test_results(parsed_cmdline_args, test_results)
  19. """
  20. import argparse
  21. import json
  22. import os
  23. class TestResult:
  24. """TestResult represents a result of executing a single test once.
  25. """
  26. def __init__(self,
  27. test_name,
  28. actual_test_result,
  29. expected_test_result='PASS'):
  30. self.test_name = test_name
  31. self.actual_test_result = actual_test_result
  32. self.expected_test_result = expected_test_result
  33. def __eq__(self, other):
  34. self_tuple = tuple(sorted(self.__dict__.items()))
  35. other_tuple = tuple(sorted(other.__dict__.items()))
  36. return self_tuple == other_tuple
  37. def __hash__(self):
  38. return hash(tuple(sorted(self.__dict__.items())))
  39. def __repr__(self):
  40. result = 'TestResult[{}: {}'.format(self.test_name,
  41. self.actual_test_result)
  42. if self.expected_test_result != 'PASS':
  43. result += ' (expecting: {})]'.format(self.expected_test_result)
  44. else:
  45. result += ']'
  46. return result
  47. def _validate_output_directory(outdir):
  48. if not os.path.isdir(outdir):
  49. raise argparse.ArgumentTypeError('No such directory: ' + outdir)
  50. return outdir
  51. def add_cmdline_args(argparse_parser):
  52. """Adds test-result-specific cmdline parameter definitions to
  53. `argparse_parser`.
  54. Args:
  55. argparse_parser: An object of argparse.ArgumentParser type.
  56. """
  57. outdir_help = 'Directory where test results will be written into.'
  58. argparse_parser.add_argument('--isolated-outdir',
  59. dest='outdir',
  60. help=outdir_help,
  61. metavar='DIRPATH',
  62. type=_validate_output_directory)
  63. outfile_help = 'If this argument is provided, then test results in the ' \
  64. 'JSON Test Results Format will be written here. See also ' \
  65. '//docs/testing/json_test_results_format.md'
  66. argparse_parser.add_argument('--isolated-script-test-output',
  67. dest='test_output',
  68. default=None,
  69. help=outfile_help,
  70. metavar='FILENAME')
  71. argparse_parser.add_argument('--isolated-script-test-perf-output',
  72. dest='ignored_perf_output',
  73. default=None,
  74. help='Deprecated and ignored.',
  75. metavar='IGNORED')
  76. def _build_json_data(list_of_test_results, seconds_since_epoch):
  77. num_failures_by_type = {}
  78. tests = {}
  79. for res in list_of_test_results:
  80. old_count = num_failures_by_type.get(res.actual_test_result, 0)
  81. num_failures_by_type[res.actual_test_result] = old_count + 1
  82. path = res.test_name.split('//')
  83. group = tests
  84. for group_name in path[:-1]:
  85. if not group_name in group:
  86. group[group_name] = {}
  87. group = group[group_name]
  88. group[path[-1]] = {
  89. 'expected': res.expected_test_result,
  90. 'actual': res.actual_test_result,
  91. }
  92. return {
  93. 'interrupted': False,
  94. 'path_delimiter': '//',
  95. 'seconds_since_epoch': seconds_since_epoch,
  96. 'version': 3,
  97. 'tests': tests,
  98. 'num_failures_by_type': num_failures_by_type,
  99. }
  100. def print_test_results(argparse_parsed_args, list_of_test_results,
  101. testing_start_time_as_seconds_since_epoch):
  102. """Prints `list_of_test_results` to a file specified on the cmdline.
  103. Args:
  104. argparse_parsed_arg: A result of an earlier call to
  105. argparse_parser.parse_args() call (where `argparse_parser` has been
  106. populated via an even earlier call to add_cmdline_args).
  107. list_of_test_results: A list of TestResult objects.
  108. testing_start_time_as_seconds_since_epoch: A number from an earlier
  109. `time.time()` call.
  110. """
  111. if argparse_parsed_args.test_output is None:
  112. return
  113. json_data = _build_json_data(list_of_test_results,
  114. testing_start_time_as_seconds_since_epoch)
  115. filepath = argparse_parsed_args.test_output
  116. with open(filepath, mode='w', encoding='utf-8') as f:
  117. json.dump(json_data, f, indent=2)