try_perf.py 4.2 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119
  1. #!/usr/bin/env python
  2. # Copyright 2014 the V8 project authors. All rights reserved.
  3. # Use of this source code is governed by a BSD-style license that can be
  4. # found in the LICENSE file.
  5. # for py2/py3 compatibility
  6. from __future__ import print_function
  7. import argparse
  8. import os
  9. import subprocess
  10. import sys
  11. BOTS = {
  12. '--linux32': 'v8_linux32_perf_try',
  13. '--linux64': 'v8_linux64_perf_try',
  14. '--nexus5': 'v8_nexus5_perf_try',
  15. '--nexus7': 'v8_nexus7_perf_try',
  16. '--pixel2': 'v8_pixel2_perf_try',
  17. }
  18. DEFAULT_BOTS = [
  19. 'v8_linux32_perf_try',
  20. 'v8_linux64_perf_try',
  21. ]
  22. PUBLIC_BENCHMARKS = [
  23. 'arewefastyet',
  24. 'ares6',
  25. 'blazor',
  26. 'compile',
  27. 'embenchen',
  28. 'emscripten',
  29. 'jetstream',
  30. 'jsbench',
  31. 'jstests',
  32. 'kraken_orig',
  33. 'massive',
  34. 'memory',
  35. 'octane',
  36. 'octane-noopt',
  37. 'octane-pr',
  38. 'octane-tf',
  39. 'octane-tf-pr',
  40. 'sunspider',
  41. 'unity',
  42. 'wasm',
  43. 'web-tooling-benchmark',
  44. ]
  45. V8_BASE = os.path.abspath(os.path.dirname(os.path.dirname(__file__)))
  46. def main():
  47. parser = argparse.ArgumentParser(description='')
  48. parser.add_argument('benchmarks', nargs='+', help='The benchmarks to run.')
  49. parser.add_argument('--extra-flags', default='',
  50. help='Extra flags to be passed to the executable.')
  51. parser.add_argument('-r', '--revision', type=str, default=None,
  52. help='Revision (use full hash!) to use for the try job; '
  53. 'default: the revision will be determined by the '
  54. 'try server; see its waterfall for more info')
  55. parser.add_argument('-v', '--verbose', action='store_true',
  56. help='Print debug information')
  57. parser.add_argument('-c', '--confidence-level', type=float,
  58. help='Repeatedly runs each benchmark until specified '
  59. 'confidence level is reached. The value is interpreted '
  60. 'as the number of standard deviations from the mean that '
  61. 'all values must lie within. Typical values are 1, 2 and '
  62. '3 and correspond to 68%%, 95%% and 99.7%% probability '
  63. 'that the measured value is within 0.1%% of the true '
  64. 'value. Larger values result in more retries and thus '
  65. 'longer runtime, but also provide more reliable results.')
  66. for option in sorted(BOTS):
  67. parser.add_argument(
  68. option, dest='bots', action='append_const', const=BOTS[option],
  69. help='Add %s trybot.' % BOTS[option])
  70. options = parser.parse_args()
  71. if not options.bots:
  72. print('No trybots specified. Using default %s.' % ','.join(DEFAULT_BOTS))
  73. options.bots = DEFAULT_BOTS
  74. if not options.benchmarks:
  75. print('Please specify the benchmarks to run as arguments.')
  76. return 1
  77. for benchmark in options.benchmarks:
  78. if benchmark not in PUBLIC_BENCHMARKS:
  79. print ('%s not found in our benchmark list. The respective trybot might '
  80. 'fail, unless you run something this script isn\'t aware of. '
  81. 'Available public benchmarks: %s' % (benchmark, PUBLIC_BENCHMARKS))
  82. print('Proceed anyways? [Y/n] ', end=' ')
  83. answer = sys.stdin.readline().strip()
  84. if answer != "" and answer != "Y" and answer != "y":
  85. return 1
  86. assert '"' not in options.extra_flags and '\'' not in options.extra_flags, (
  87. 'Invalid flag specification.')
  88. # Ensure depot_tools are updated.
  89. subprocess.check_output(
  90. 'update_depot_tools', shell=True, stderr=subprocess.STDOUT, cwd=V8_BASE)
  91. cmd = ['git cl try', '-B', 'luci.v8-internal.try']
  92. cmd += ['-b %s' % bot for bot in options.bots]
  93. if options.revision:
  94. cmd.append('-r %s' % options.revision)
  95. benchmarks = ['"%s"' % benchmark for benchmark in options.benchmarks]
  96. cmd.append('-p \'testfilter=[%s]\'' % ','.join(benchmarks))
  97. if options.extra_flags:
  98. cmd.append('-p \'extra_flags="%s"\'' % options.extra_flags)
  99. if options.confidence_level:
  100. cmd.append('-p confidence_level=%f' % options.confidence_level)
  101. if options.verbose:
  102. cmd.append('-vv')
  103. print('Running %s' % ' '.join(cmd))
  104. subprocess.check_call(' '.join(cmd), shell=True, cwd=V8_BASE)
  105. if __name__ == '__main__': # pragma: no cover
  106. sys.exit(main())