123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960 |
- #!/usr/bin/env python
- # Copyright 2014 The Chromium Authors. All rights reserved.
- # Use of this source code is governed by a BSD-style license that can be
- # found in the LICENSE file.
- """Runs a test repeatedly to measure its flakiness. The return code is non-zero
- if the failure rate is higher than the specified threshold, but is not 100%."""
- from __future__ import print_function
- import argparse
- import multiprocessing.dummy
- import subprocess
- import sys
- import time
- def load_options():
- parser = argparse.ArgumentParser(description=__doc__)
- parser.add_argument('--retries', default=1000, type=int,
- help='Number of test retries to measure flakiness.')
- parser.add_argument('--threshold', default=0.05, type=float,
- help='Minimum flakiness level at which test is '
- 'considered flaky.')
- parser.add_argument('--jobs', '-j', type=int, default=1,
- help='Number of parallel jobs to run tests.')
- parser.add_argument('command', nargs='+', help='Command to run test.')
- return parser.parse_args()
- def run_test(job):
- print('Starting retry attempt %d out of %d' % (job['index'] + 1,
- job['retries']))
- return subprocess.check_call(job['cmd'], stdout=subprocess.PIPE,
- stderr=subprocess.STDOUT)
- def main():
- options = load_options()
- num_passed = num_failed = 0
- running = []
- pool = multiprocessing.dummy.Pool(processes=options.jobs)
- args = [{'index': index, 'retries': options.retries, 'cmd': options.command}
- for index in range(options.retries)]
- results = pool.map(run_test, args)
- num_passed = len([retcode for retcode in results if retcode == 0])
- num_failed = len(results) - num_passed
- if num_passed == 0:
- flakiness = 0
- else:
- flakiness = num_failed / float(len(results))
- print('Flakiness is %.2f' % flakiness)
- if flakiness > options.threshold:
- return 1
- else:
- return 0
- if __name__ == '__main__':
- sys.exit(main())
|