# Copyright 2021 The Chromium Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. """This is a library for working with test executables in a way that is Chromium-bot-friendly as specified by //docs/testing/test_executable_api.md Example usage: import os import sys import main_program import rust_main_program if __name__ == '__main__': cmdline_parser = argparse.ArgumentParser() main_program.add_cmdline_args(cmdline_parser) ... adding other cmdline parameter definitions ... parsed_cmdline_args = cmdline_parser.parse_args() test_executable_wrappers = [] test_executable_wrappers.append( rust_main_program.TestExecutableWrapper(...)) ... main_program.main( test_executable_wrappers, parsed_cmdline_args, os.environ) """ import argparse import time import test_filtering import test_results def add_cmdline_args(argparse_parser): """Adds test-filtering-specific cmdline parameter definitions to `argparse_parser`. Args: argparse_parser: An object of argparse.ArgumentParser type. """ test_filtering.add_cmdline_args(argparse_parser) test_results.add_cmdline_args(argparse_parser) argparse_parser.add_argument( '--isolated-script-test-launcher-retry-limit', dest='retry_limit', default=3, help='Sets the limit of test retries on failures to N.', metavar='N', type=int) argparse_parser.add_argument('--isolated-script-test-repeat', dest='repetitions', default=1, help='Repeats each test N times.', metavar='N', type=int) def _calculate_tests_to_run(argparse_parsed_args, env, test_executable_wrappers): tests = [] for wrapper in test_executable_wrappers: extra_tests = wrapper.list_all_tests() for extra_test in extra_tests: assert extra_test not in tests tests.extend(extra_tests) return test_filtering.filter_tests(argparse_parsed_args, env, tests) def _run_tests_and_save_results(argparse_parsed_args, list_of_tests_to_run, test_executable_wrapper): start_time = time.time() results = [] for wrapper in test_executable_wrapper: results.extend(wrapper.run_tests(list_of_tests_to_run)) test_results.print_test_results(argparse_parsed_args, results, start_time) def main(test_executable_wrappers, argparse_parsed_args, env): """Runs tests within `test_executable_wrappers` using cmdline arguments and environment variables to figure out 1) which subset of tests to run, 2) where to save the JSON file with test results. Args: test_executable_wrappers: A list of objects providing list_all_tests(...) and run_tests(...) methods (see rust_main_program._TestExecutableWrapper). argparse_parsed_arg: A result of an earlier call to argparse_parser.parse_args() call (where `argparse_parser` has been populated via an even earlier call to add_cmdline_args). env: a dictionary-like object (typically from `os.environ`). """ list_of_test_names_to_run = _calculate_tests_to_run( argparse_parsed_args, env, test_executable_wrappers) _run_tests_and_save_results(argparse_parsed_args, list_of_test_names_to_run, test_executable_wrappers) # TODO(lukasza): Repeat tests `args.repetitions` times. # TODO(lukasza): Retry failing times up to `args.retry_limit` times.