skpbench.py 13 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363
  1. #!/usr/bin/env python
  2. # Copyright 2016 Google Inc.
  3. #
  4. # Use of this source code is governed by a BSD-style license that can be
  5. # found in the LICENSE file.
  6. from __future__ import print_function
  7. from _adb import Adb
  8. from _benchresult import BenchResult
  9. from _hardware import HardwareException, Hardware
  10. from argparse import ArgumentParser
  11. from multiprocessing import Queue
  12. from threading import Thread, Timer
  13. import collections
  14. import glob
  15. import math
  16. import re
  17. import subprocess
  18. import sys
  19. import time
  20. __argparse = ArgumentParser(description="""
  21. Executes the skpbench binary with various configs and skps.
  22. Also monitors the output in order to filter out and re-run results that have an
  23. unacceptable stddev.
  24. """)
  25. __argparse.add_argument('skpbench',
  26. help="path to the skpbench binary")
  27. __argparse.add_argument('--adb',
  28. action='store_true', help="execute skpbench over adb")
  29. __argparse.add_argument('--adb_binary', default='adb',
  30. help="The name of the adb binary to use.")
  31. __argparse.add_argument('-s', '--device-serial',
  32. help="if using adb, ID of the specific device to target "
  33. "(only required if more than 1 device is attached)")
  34. __argparse.add_argument('-m', '--max-stddev',
  35. type=float, default=4,
  36. help="initial max allowable relative standard deviation")
  37. __argparse.add_argument('-x', '--suffix',
  38. help="suffix to append on config (e.g. '_before', '_after')")
  39. __argparse.add_argument('-w','--write-path',
  40. help="directory to save .png proofs to disk.")
  41. __argparse.add_argument('-v','--verbosity',
  42. type=int, default=1, help="level of verbosity (0=none to 5=debug)")
  43. __argparse.add_argument('-d', '--duration',
  44. type=int, help="number of milliseconds to run each benchmark")
  45. __argparse.add_argument('-l', '--sample-ms',
  46. type=int, help="duration of a sample (minimum)")
  47. __argparse.add_argument('--gpu',
  48. action='store_true',
  49. help="perform timing on the gpu clock instead of cpu (gpu work only)")
  50. __argparse.add_argument('--fps',
  51. action='store_true', help="use fps instead of ms")
  52. __argparse.add_argument('--pr',
  53. help="comma- or space-separated list of GPU path renderers, including: "
  54. "[[~]all [~]default [~]dashline [~]nvpr [~]msaa [~]aaconvex "
  55. "[~]aalinearizing [~]small [~]tess]")
  56. __argparse.add_argument('--cc',
  57. action='store_true', help="allow coverage counting shortcuts to render paths")
  58. __argparse.add_argument('--nocache',
  59. action='store_true', help="disable caching of path mask textures")
  60. __argparse.add_argument('-c', '--config',
  61. default='gl', help="comma- or space-separated list of GPU configs")
  62. __argparse.add_argument('-a', '--resultsfile',
  63. help="optional file to append results into")
  64. __argparse.add_argument('--ddl',
  65. action='store_true', help="record the skp into DDLs before rendering")
  66. __argparse.add_argument('--ddlNumAdditionalThreads',
  67. type=int, default=0,
  68. help="number of DDL recording threads in addition to main one")
  69. __argparse.add_argument('--ddlTilingWidthHeight',
  70. type=int, default=0, help="number of tiles along one edge when in DDL mode")
  71. __argparse.add_argument('--ddlRecordTime',
  72. action='store_true', help="report just the cpu time spent recording DDLs")
  73. __argparse.add_argument('--gpuThreads',
  74. type=int, default=-1,
  75. help="Create this many extra threads to assist with GPU work, including"
  76. " software path rendering. Defaults to two.")
  77. __argparse.add_argument('srcs',
  78. nargs='+',
  79. help=".skp files or directories to expand for .skp files, and/or .svg files")
  80. FLAGS = __argparse.parse_args()
  81. if FLAGS.adb:
  82. import _adb_path as _path
  83. _path.init(FLAGS.device_serial, FLAGS.adb_binary)
  84. else:
  85. import _os_path as _path
  86. def dump_commandline_if_verbose(commandline):
  87. if FLAGS.verbosity >= 5:
  88. quoted = ['\'%s\'' % re.sub(r'([\\\'])', r'\\\1', x) for x in commandline]
  89. print(' '.join(quoted), file=sys.stderr)
  90. class StddevException(Exception):
  91. pass
  92. class Message:
  93. READLINE = 0,
  94. POLL_HARDWARE = 1,
  95. EXIT = 2
  96. def __init__(self, message, value=None):
  97. self.message = message
  98. self.value = value
  99. class SubprocessMonitor(Thread):
  100. def __init__(self, queue, proc):
  101. self._queue = queue
  102. self._proc = proc
  103. Thread.__init__(self)
  104. def run(self):
  105. """Runs on the background thread."""
  106. for line in iter(self._proc.stdout.readline, b''):
  107. self._queue.put(Message(Message.READLINE, line.decode('utf-8').rstrip()))
  108. self._queue.put(Message(Message.EXIT))
  109. class SKPBench:
  110. ARGV = [FLAGS.skpbench, '--verbosity', str(FLAGS.verbosity)]
  111. if FLAGS.duration:
  112. ARGV.extend(['--duration', str(FLAGS.duration)])
  113. if FLAGS.sample_ms:
  114. ARGV.extend(['--sampleMs', str(FLAGS.sample_ms)])
  115. if FLAGS.gpu:
  116. ARGV.extend(['--gpuClock', 'true'])
  117. if FLAGS.fps:
  118. ARGV.extend(['--fps', 'true'])
  119. if FLAGS.pr:
  120. ARGV.extend(['--pr'] + re.split(r'[ ,]', FLAGS.pr))
  121. if FLAGS.cc:
  122. ARGV.extend(['--cc', 'true'])
  123. if FLAGS.nocache:
  124. ARGV.extend(['--cachePathMasks', 'false'])
  125. if FLAGS.gpuThreads != -1:
  126. ARGV.extend(['--gpuThreads', str(FLAGS.gpuThreads)])
  127. # DDL parameters
  128. if FLAGS.ddl:
  129. ARGV.extend(['--ddl', 'true'])
  130. if FLAGS.ddlNumAdditionalThreads:
  131. ARGV.extend(['--ddlNumAdditionalThreads',
  132. str(FLAGS.ddlNumAdditionalThreads)])
  133. if FLAGS.ddlTilingWidthHeight:
  134. ARGV.extend(['--ddlTilingWidthHeight', str(FLAGS.ddlTilingWidthHeight)])
  135. if FLAGS.ddlRecordTime:
  136. ARGV.extend(['--ddlRecordTime', 'true'])
  137. if FLAGS.adb:
  138. if FLAGS.device_serial is None:
  139. ARGV[:0] = [FLAGS.adb_binary, 'shell']
  140. else:
  141. ARGV[:0] = [FLAGS.adb_binary, '-s', FLAGS.device_serial, 'shell']
  142. @classmethod
  143. def get_header(cls, outfile=sys.stdout):
  144. commandline = cls.ARGV + ['--duration', '0']
  145. dump_commandline_if_verbose(commandline)
  146. out = subprocess.check_output(commandline, stderr=subprocess.STDOUT)
  147. return out.rstrip()
  148. @classmethod
  149. def run_warmup(cls, warmup_time, config):
  150. if not warmup_time:
  151. return
  152. print('running %i second warmup...' % warmup_time, file=sys.stderr)
  153. commandline = cls.ARGV + ['--duration', str(warmup_time * 1000),
  154. '--config', config,
  155. '--src', 'warmup']
  156. dump_commandline_if_verbose(commandline)
  157. output = subprocess.check_output(commandline, stderr=subprocess.STDOUT)
  158. # validate the warmup run output.
  159. for line in output.decode('utf-8').split('\n'):
  160. match = BenchResult.match(line.rstrip())
  161. if match and match.bench == 'warmup':
  162. return
  163. raise Exception('Invalid warmup output:\n%s' % output)
  164. def __init__(self, src, config, max_stddev, best_result=None):
  165. self.src = src
  166. self.config = config
  167. self.max_stddev = max_stddev
  168. self.best_result = best_result
  169. self._queue = Queue()
  170. self._proc = None
  171. self._monitor = None
  172. self._hw_poll_timer = None
  173. def __enter__(self):
  174. return self
  175. def __exit__(self, exception_type, exception_value, traceback):
  176. if self._proc:
  177. self.terminate()
  178. if self._hw_poll_timer:
  179. self._hw_poll_timer.cancel()
  180. def execute(self, hardware):
  181. hardware.sanity_check()
  182. self._schedule_hardware_poll()
  183. commandline = self.ARGV + ['--config', self.config,
  184. '--src', self.src,
  185. '--suppressHeader', 'true']
  186. if FLAGS.write_path:
  187. pngfile = _path.join(FLAGS.write_path, self.config,
  188. _path.basename(self.src) + '.png')
  189. commandline.extend(['--png', pngfile])
  190. dump_commandline_if_verbose(commandline)
  191. self._proc = subprocess.Popen(commandline, stdout=subprocess.PIPE,
  192. stderr=subprocess.STDOUT)
  193. self._monitor = SubprocessMonitor(self._queue, self._proc)
  194. self._monitor.start()
  195. while True:
  196. message = self._queue.get()
  197. if message.message == Message.READLINE:
  198. result = BenchResult.match(message.value)
  199. if result:
  200. hardware.sanity_check()
  201. self._process_result(result)
  202. elif hardware.filter_line(message.value):
  203. print(message.value, file=sys.stderr)
  204. continue
  205. if message.message == Message.POLL_HARDWARE:
  206. hardware.sanity_check()
  207. self._schedule_hardware_poll()
  208. continue
  209. if message.message == Message.EXIT:
  210. self._monitor.join()
  211. self._proc.wait()
  212. if self._proc.returncode != 0:
  213. raise Exception("skpbench exited with nonzero exit code %i" %
  214. self._proc.returncode)
  215. self._proc = None
  216. break
  217. def _schedule_hardware_poll(self):
  218. if self._hw_poll_timer:
  219. self._hw_poll_timer.cancel()
  220. self._hw_poll_timer = \
  221. Timer(1, lambda: self._queue.put(Message(Message.POLL_HARDWARE)))
  222. self._hw_poll_timer.start()
  223. def _process_result(self, result):
  224. if not self.best_result or result.stddev <= self.best_result.stddev:
  225. self.best_result = result
  226. elif FLAGS.verbosity >= 2:
  227. print("reusing previous result for %s/%s with lower stddev "
  228. "(%s%% instead of %s%%)." %
  229. (result.config, result.bench, self.best_result.stddev,
  230. result.stddev), file=sys.stderr)
  231. if self.max_stddev and self.best_result.stddev > self.max_stddev:
  232. raise StddevException()
  233. def terminate(self):
  234. if self._proc:
  235. self._proc.terminate()
  236. self._monitor.join()
  237. self._proc.wait()
  238. self._proc = None
  239. def emit_result(line, resultsfile=None):
  240. print(line)
  241. sys.stdout.flush()
  242. if resultsfile:
  243. print(line, file=resultsfile)
  244. resultsfile.flush()
  245. def run_benchmarks(configs, srcs, hardware, resultsfile=None):
  246. hasheader = False
  247. benches = collections.deque([(src, config, FLAGS.max_stddev)
  248. for src in srcs
  249. for config in configs])
  250. while benches:
  251. try:
  252. with hardware:
  253. SKPBench.run_warmup(hardware.warmup_time, configs[0])
  254. if not hasheader:
  255. emit_result(SKPBench.get_header(), resultsfile)
  256. hasheader = True
  257. while benches:
  258. benchargs = benches.popleft()
  259. with SKPBench(*benchargs) as skpbench:
  260. try:
  261. skpbench.execute(hardware)
  262. if skpbench.best_result:
  263. emit_result(skpbench.best_result.format(FLAGS.suffix),
  264. resultsfile)
  265. else:
  266. print("WARNING: no result for %s with config %s" %
  267. (skpbench.src, skpbench.config), file=sys.stderr)
  268. except StddevException:
  269. retry_max_stddev = skpbench.max_stddev * math.sqrt(2)
  270. if FLAGS.verbosity >= 1:
  271. print("stddev is too high for %s/%s (%s%%, max=%.2f%%), "
  272. "re-queuing with max=%.2f%%." %
  273. (skpbench.best_result.config, skpbench.best_result.bench,
  274. skpbench.best_result.stddev, skpbench.max_stddev,
  275. retry_max_stddev),
  276. file=sys.stderr)
  277. benches.append((skpbench.src, skpbench.config, retry_max_stddev,
  278. skpbench.best_result))
  279. except HardwareException as exception:
  280. skpbench.terminate()
  281. if FLAGS.verbosity >= 4:
  282. hardware.print_debug_diagnostics()
  283. if FLAGS.verbosity >= 1:
  284. print("%s; rebooting and taking a %i second nap..." %
  285. (exception.message, exception.sleeptime), file=sys.stderr)
  286. benches.appendleft(benchargs) # retry the same bench next time.
  287. raise # wake hw up from benchmarking mode before the nap.
  288. except HardwareException as exception:
  289. time.sleep(exception.sleeptime)
  290. def main():
  291. # Delimiter is ',' or ' ', skip if nested inside parens (e.g. gpu(a=b,c=d)).
  292. DELIMITER = r'[, ](?!(?:[^(]*\([^)]*\))*[^()]*\))'
  293. configs = re.split(DELIMITER, FLAGS.config)
  294. srcs = _path.find_skps(FLAGS.srcs)
  295. if FLAGS.adb:
  296. adb = Adb(FLAGS.device_serial, FLAGS.adb_binary,
  297. echo=(FLAGS.verbosity >= 5))
  298. model = adb.check('getprop ro.product.model').strip()
  299. if model == 'Pixel C':
  300. from _hardware_pixel_c import HardwarePixelC
  301. hardware = HardwarePixelC(adb)
  302. elif model == 'Pixel':
  303. from _hardware_pixel import HardwarePixel
  304. hardware = HardwarePixel(adb)
  305. elif model == 'Pixel 2':
  306. from _hardware_pixel2 import HardwarePixel2
  307. hardware = HardwarePixel2(adb)
  308. elif model == 'Nexus 6P':
  309. from _hardware_nexus_6p import HardwareNexus6P
  310. hardware = HardwareNexus6P(adb)
  311. else:
  312. from _hardware_android import HardwareAndroid
  313. print("WARNING: %s: don't know how to monitor this hardware; results "
  314. "may be unreliable." % model, file=sys.stderr)
  315. hardware = HardwareAndroid(adb)
  316. else:
  317. hardware = Hardware()
  318. if FLAGS.resultsfile:
  319. with open(FLAGS.resultsfile, mode='a+') as resultsfile:
  320. run_benchmarks(configs, srcs, hardware, resultsfile=resultsfile)
  321. else:
  322. run_benchmarks(configs, srcs, hardware)
  323. if __name__ == '__main__':
  324. main()