is_flaky.py 2.0 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960
  1. #!/usr/bin/env python
  2. # Copyright 2014 The Chromium Authors. All rights reserved.
  3. # Use of this source code is governed by a BSD-style license that can be
  4. # found in the LICENSE file.
  5. """Runs a test repeatedly to measure its flakiness. The return code is non-zero
  6. if the failure rate is higher than the specified threshold, but is not 100%."""
  7. from __future__ import print_function
  8. import argparse
  9. import multiprocessing.dummy
  10. import subprocess
  11. import sys
  12. import time
  13. def load_options():
  14. parser = argparse.ArgumentParser(description=__doc__)
  15. parser.add_argument('--retries', default=1000, type=int,
  16. help='Number of test retries to measure flakiness.')
  17. parser.add_argument('--threshold', default=0.05, type=float,
  18. help='Minimum flakiness level at which test is '
  19. 'considered flaky.')
  20. parser.add_argument('--jobs', '-j', type=int, default=1,
  21. help='Number of parallel jobs to run tests.')
  22. parser.add_argument('command', nargs='+', help='Command to run test.')
  23. return parser.parse_args()
  24. def run_test(job):
  25. print('Starting retry attempt %d out of %d' % (job['index'] + 1,
  26. job['retries']))
  27. return subprocess.check_call(job['cmd'], stdout=subprocess.PIPE,
  28. stderr=subprocess.STDOUT)
  29. def main():
  30. options = load_options()
  31. num_passed = num_failed = 0
  32. running = []
  33. pool = multiprocessing.dummy.Pool(processes=options.jobs)
  34. args = [{'index': index, 'retries': options.retries, 'cmd': options.command}
  35. for index in range(options.retries)]
  36. results = pool.map(run_test, args)
  37. num_passed = len([retcode for retcode in results if retcode == 0])
  38. num_failed = len(results) - num_passed
  39. if num_passed == 0:
  40. flakiness = 0
  41. else:
  42. flakiness = num_failed / float(len(results))
  43. print('Flakiness is %.2f' % flakiness)
  44. if flakiness > options.threshold:
  45. return 1
  46. else:
  47. return 0
  48. if __name__ == '__main__':
  49. sys.exit(main())