main_program.py 3.7 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101
  1. # Copyright 2021 The Chromium Authors. All rights reserved.
  2. # Use of this source code is governed by a BSD-style license that can be
  3. # found in the LICENSE file.
  4. """This is a library for working with test executables in a way that is
  5. Chromium-bot-friendly as specified by //docs/testing/test_executable_api.md
  6. Example usage:
  7. import os
  8. import sys
  9. import main_program
  10. import rust_main_program
  11. if __name__ == '__main__':
  12. cmdline_parser = argparse.ArgumentParser()
  13. main_program.add_cmdline_args(cmdline_parser)
  14. ... adding other cmdline parameter definitions ...
  15. parsed_cmdline_args = cmdline_parser.parse_args()
  16. test_executable_wrappers = []
  17. test_executable_wrappers.append(
  18. rust_main_program.TestExecutableWrapper(...))
  19. ...
  20. main_program.main(
  21. test_executable_wrappers, parsed_cmdline_args, os.environ)
  22. """
  23. import argparse
  24. import time
  25. import test_filtering
  26. import test_results
  27. def add_cmdline_args(argparse_parser):
  28. """Adds test-filtering-specific cmdline parameter definitions to
  29. `argparse_parser`.
  30. Args:
  31. argparse_parser: An object of argparse.ArgumentParser type.
  32. """
  33. test_filtering.add_cmdline_args(argparse_parser)
  34. test_results.add_cmdline_args(argparse_parser)
  35. argparse_parser.add_argument(
  36. '--isolated-script-test-launcher-retry-limit',
  37. dest='retry_limit',
  38. default=3,
  39. help='Sets the limit of test retries on failures to N.',
  40. metavar='N',
  41. type=int)
  42. argparse_parser.add_argument('--isolated-script-test-repeat',
  43. dest='repetitions',
  44. default=1,
  45. help='Repeats each test N times.',
  46. metavar='N',
  47. type=int)
  48. def _calculate_tests_to_run(argparse_parsed_args, env,
  49. test_executable_wrappers):
  50. tests = []
  51. for wrapper in test_executable_wrappers:
  52. extra_tests = wrapper.list_all_tests()
  53. for extra_test in extra_tests:
  54. assert extra_test not in tests
  55. tests.extend(extra_tests)
  56. return test_filtering.filter_tests(argparse_parsed_args, env, tests)
  57. def _run_tests_and_save_results(argparse_parsed_args, list_of_tests_to_run,
  58. test_executable_wrapper):
  59. start_time = time.time()
  60. results = []
  61. for wrapper in test_executable_wrapper:
  62. results.extend(wrapper.run_tests(list_of_tests_to_run))
  63. test_results.print_test_results(argparse_parsed_args, results, start_time)
  64. def main(test_executable_wrappers, argparse_parsed_args, env):
  65. """Runs tests within `test_executable_wrappers` using cmdline arguments and
  66. environment variables to figure out 1) which subset of tests to run, 2)
  67. where to save the JSON file with test results.
  68. Args:
  69. test_executable_wrappers: A list of objects providing
  70. list_all_tests(...) and run_tests(...) methods (see
  71. rust_main_program._TestExecutableWrapper).
  72. argparse_parsed_arg: A result of an earlier call to
  73. argparse_parser.parse_args() call (where `argparse_parser` has been
  74. populated via an even earlier call to add_cmdline_args).
  75. env: a dictionary-like object (typically from `os.environ`).
  76. """
  77. list_of_test_names_to_run = _calculate_tests_to_run(
  78. argparse_parsed_args, env, test_executable_wrappers)
  79. _run_tests_and_save_results(argparse_parsed_args,
  80. list_of_test_names_to_run,
  81. test_executable_wrappers)
  82. # TODO(lukasza): Repeat tests `args.repetitions` times.
  83. # TODO(lukasza): Retry failing times up to `args.retry_limit` times.