perry.py 2.9 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109
  1. #!/usr/bin/env python
  2. # Copyright 2017 The Chromium Authors. All rights reserved.
  3. # Use of this source code is governed by a BSD-style license that can be
  4. # found in the LICENSE file.
  5. """Runs all permutations of pairs of tests in a gtest binary to attempt to
  6. detect state leakage between tests.
  7. Example invocation:
  8. gn gen out/asan --args='is_asan=true enable_nacl=false is_debug=false'
  9. ninja -C out/asan base_unittests
  10. tools/perry.py out/asan/base_unittests > perry.log &
  11. tail -f perry.log
  12. You might want to run it in `screen` as it'll take a while.
  13. """
  14. from __future__ import print_function
  15. import argparse
  16. import os
  17. import multiprocessing
  18. import subprocess
  19. import sys
  20. def _GetTestList(path_to_binary):
  21. """Returns a set of full test names.
  22. Each test will be of the form "Case.Test". There will be a separate line
  23. for each combination of Case/Test (there are often multiple tests in each
  24. case).
  25. """
  26. raw_output = subprocess.check_output([path_to_binary, "--gtest_list_tests"])
  27. input_lines = raw_output.splitlines()
  28. # The format of the gtest_list_tests output is:
  29. # "Case1."
  30. # " Test1 # <Optional extra stuff>"
  31. # " Test2"
  32. # "Case2."
  33. # " Test1"
  34. case_name = '' # Includes trailing dot.
  35. test_set = set()
  36. for line in input_lines:
  37. if len(line) > 1:
  38. if '#' in line:
  39. line = line[:line.find('#')]
  40. if line[0] == ' ':
  41. # Indented means a test in previous case.
  42. test_set.add(case_name + line.strip())
  43. else:
  44. # New test case.
  45. case_name = line.strip()
  46. return test_set
  47. def _CheckForFailure(data):
  48. test_binary, pair0, pair1 = data
  49. p = subprocess.Popen(
  50. [test_binary, '--gtest_repeat=5', '--gtest_shuffle',
  51. '--gtest_filter=' + pair0 + ':' + pair1],
  52. stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
  53. out, _ = p.communicate()
  54. if p.returncode != 0:
  55. return (pair0, pair1, out)
  56. return None
  57. def _PrintStatus(i, total, failed):
  58. status = '%d of %d tested (%d failures)' % (i+1, total, failed)
  59. print('\r%s%s' % (status, '\x1B[K'), end=' ')
  60. sys.stdout.flush()
  61. def main():
  62. parser = argparse.ArgumentParser(description="Find failing pairs of tests.")
  63. parser.add_argument('binary', help='Path to gtest binary or wrapper script.')
  64. args = parser.parse_args()
  65. print('Getting test list...')
  66. all_tests = _GetTestList(args.binary)
  67. permuted = [(args.binary, x, y) for x in all_tests for y in all_tests]
  68. failed = []
  69. pool = multiprocessing.Pool()
  70. total_count = len(permuted)
  71. for i, result in enumerate(pool.imap_unordered(
  72. _CheckForFailure, permuted, 1)):
  73. if result:
  74. print('\n--gtest_filter=%s:%s failed\n\n%s\n\n' % (result[0], result[1],
  75. result[2]))
  76. failed.append(result)
  77. _PrintStatus(i, total_count, len(failed))
  78. pool.terminate()
  79. pool.join()
  80. if failed:
  81. print('Failed pairs:')
  82. for f in failed:
  83. print(f[0], f[1])
  84. return 0
  85. if __name__ == '__main__':
  86. sys.exit(main())