generate-runtime-call-stats.py 14 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490
  1. #!/usr/bin/python3
  2. # Copyright 2019 the V8 project authors. All rights reserved.
  3. # Use of this source code is governed by a BSD-style license that can be
  4. # found in the LICENSE file.
  5. # Runs chromium/src/run_benchmark for a given story and extracts the generated
  6. # runtime call stats.
  7. import argparse
  8. import csv
  9. import json
  10. import glob
  11. import os
  12. import pathlib
  13. import re
  14. import tabulate
  15. import shutil
  16. import statistics
  17. import subprocess
  18. import sys
  19. import tempfile
  20. from callstats_groups import RUNTIME_CALL_STATS_GROUPS
  21. JSON_FILE_EXTENSION=".pb_converted.json"
  22. def parse_args():
  23. parser = argparse.ArgumentParser(
  24. description="Run story and collect runtime call stats.")
  25. parser.add_argument("story", metavar="story", nargs=1, help="story to run")
  26. parser.add_argument(
  27. "--group",
  28. dest="group",
  29. action="store_true",
  30. help="group common stats together into buckets")
  31. parser.add_argument(
  32. "-r",
  33. "--repeats",
  34. dest="repeats",
  35. metavar="N",
  36. action="store",
  37. type=int,
  38. default=1,
  39. help="number of times to run the story")
  40. parser.add_argument(
  41. "-v",
  42. "--verbose",
  43. dest="verbose",
  44. action="store_true",
  45. help="output benchmark runs to stdout")
  46. parser.add_argument(
  47. "--device",
  48. dest="device",
  49. action="store",
  50. help="device to run the test on. Passed directly to run_benchmark")
  51. parser.add_argument(
  52. "-d",
  53. "--dir",
  54. dest="dir",
  55. action="store",
  56. help=("directory to look for already generated output in. This must "
  57. "already exists and it won't re-run the benchmark"))
  58. parser.add_argument(
  59. "-f",
  60. "--format",
  61. dest="format",
  62. action="store",
  63. choices=["csv", "table"],
  64. help="output as CSV")
  65. parser.add_argument(
  66. "-o",
  67. "--output",
  68. metavar="FILE",
  69. dest="out_file",
  70. action="store",
  71. help="write table to FILE rather stdout")
  72. parser.add_argument(
  73. "--browser",
  74. dest="browser",
  75. metavar="BROWSER_TYPE",
  76. action="store",
  77. default="release",
  78. help=("Passed directly to --browser option of run_benchmark. Ignored if "
  79. "-executable is used"))
  80. parser.add_argument(
  81. "-e",
  82. "--executable",
  83. dest="executable",
  84. metavar="EXECUTABLE",
  85. action="store",
  86. help=("path to executable to run. If not given it will pass '--browser "
  87. "release' to run_benchmark"))
  88. parser.add_argument(
  89. "--chromium-dir",
  90. dest="chromium_dir",
  91. metavar="DIR",
  92. action="store",
  93. default=".",
  94. help=("path to chromium directory. If not given, the script must be run "
  95. "inside the chromium/src directory"))
  96. parser.add_argument(
  97. "--js-flags", dest="js_flags", action="store", help="flags to pass to v8")
  98. parser.add_argument(
  99. "--extra-browser-args",
  100. dest="browser_args",
  101. action="store",
  102. help="flags to pass to chrome")
  103. parser.add_argument(
  104. "--benchmark",
  105. dest="benchmark",
  106. action="store",
  107. default="v8.browsing_desktop",
  108. help="benchmark to run")
  109. parser.add_argument(
  110. "--stdev",
  111. dest="stdev",
  112. action="store_true",
  113. help="adds columns for the standard deviation")
  114. parser.add_argument(
  115. "--filter",
  116. dest="filter",
  117. action="append",
  118. help="useable with --group to only show buckets specified by filter")
  119. parser.add_argument(
  120. "--retain",
  121. dest="retain",
  122. action="store",
  123. default="json",
  124. choices=["none", "json", "all"],
  125. help=("controls artifacts to be retained after run. With none, all files "
  126. "are deleted; only the json.gz file is retained for each run; and "
  127. "all keep all files"))
  128. return parser.parse_args()
  129. def process_trace(trace_file):
  130. text_string = pathlib.Path(trace_file).read_text()
  131. result = json.loads(text_string)
  132. output = {}
  133. result = result["traceEvents"]
  134. for o in result:
  135. o = o["args"]
  136. if "runtime-call-stats" in o:
  137. r = o["runtime-call-stats"]
  138. for name in r:
  139. count = r[name][0]
  140. duration = r[name][1]
  141. if name in output:
  142. output[name]["count"] += count
  143. output[name]["duration"] += duration
  144. else:
  145. output[name] = {"count": count, "duration": duration}
  146. return output
  147. def run_benchmark(story,
  148. repeats=1,
  149. output_dir=".",
  150. verbose=False,
  151. js_flags=None,
  152. browser_args=None,
  153. chromium_dir=".",
  154. executable=None,
  155. benchmark="v8.browsing_desktop",
  156. device=None,
  157. browser="release"):
  158. orig_chromium_dir = chromium_dir
  159. xvfb = os.path.join(chromium_dir, "testing", "xvfb.py")
  160. if not os.path.isfile(xvfb):
  161. chromium_dir = os.path(chromium_dir, "src")
  162. xvfb = os.path.join(chromium_dir, "testing", "xvfb.py")
  163. if not os.path.isfile(xvfb):
  164. print(("chromium_dir does not point to a valid chromium checkout: " +
  165. orig_chromium_dir))
  166. sys.exit(1)
  167. command = [
  168. xvfb,
  169. os.path.join(chromium_dir, "tools", "perf", "run_benchmark"),
  170. "run",
  171. "--story",
  172. story,
  173. "--pageset-repeat",
  174. str(repeats),
  175. "--output-dir",
  176. output_dir,
  177. "--intermediate-dir",
  178. os.path.join(output_dir, "artifacts"),
  179. benchmark,
  180. ]
  181. if executable:
  182. command += ["--browser-executable", executable]
  183. else:
  184. command += ["--browser", browser]
  185. if device:
  186. command += ["--device", device]
  187. if browser_args:
  188. command += ["--extra-browser-args", browser_args]
  189. if js_flags:
  190. command += ["--js-flags", js_flags]
  191. if not benchmark.startswith("v8."):
  192. # Most benchmarks by default don't collect runtime call stats so enable them
  193. # manually.
  194. categories = [
  195. "v8",
  196. "disabled-by-default-v8.runtime_stats",
  197. ]
  198. command += ["--extra-chrome-categories", ",".join(categories)]
  199. print("Output directory: %s" % output_dir)
  200. stdout = ""
  201. print(f"Running: {' '.join(command)}\n")
  202. proc = subprocess.Popen(
  203. command,
  204. stdout=subprocess.PIPE,
  205. stderr=subprocess.PIPE,
  206. universal_newlines=True)
  207. proc.stderr.close()
  208. status_matcher = re.compile(r"\[ +(\w+) +\]")
  209. for line in iter(proc.stdout.readline, ""):
  210. stdout += line
  211. match = status_matcher.match(line)
  212. if verbose or match:
  213. print(line, end="")
  214. proc.stdout.close()
  215. if proc.wait() != 0:
  216. print("\nrun_benchmark failed:")
  217. # If verbose then everything has already been printed.
  218. if not verbose:
  219. print(stdout)
  220. sys.exit(1)
  221. print("\nrun_benchmark completed")
  222. def write_output(f, table, headers, run_count, format="table"):
  223. if format == "csv":
  224. # strip new lines from CSV output
  225. headers = [h.replace("\n", " ") for h in headers]
  226. writer = csv.writer(f)
  227. writer.writerow(headers)
  228. writer.writerows(table)
  229. else:
  230. # First column is name, and then they alternate between counts and durations
  231. summary_count = len(headers) - 2 * run_count - 1
  232. floatfmt = ("",) + (".0f", ".2f") * run_count + (".2f",) * summary_count
  233. f.write(tabulate.tabulate(table, headers=headers, floatfmt=floatfmt))
  234. f.write("\n")
  235. class Row:
  236. def __init__(self, name, run_count):
  237. self.name = name
  238. self.durations = [0] * run_count
  239. self.counts = [0] * run_count
  240. self.mean_duration = None
  241. self.mean_count = None
  242. self.stdev_duration = None
  243. self.stdev_count = None
  244. def __repr__(self):
  245. data_str = ", ".join(
  246. str((c, d)) for (c, d) in zip(self.counts, self.durations))
  247. return (f"{self.name}: {data_str}, mean_count: {self.mean_count}, " +
  248. f"mean_duration: {self.mean_duration}")
  249. def add_data(self, counts, durations):
  250. self.counts = counts
  251. self.durations = durations
  252. def add_data_point(self, run, count, duration):
  253. self.counts[run] = count
  254. self.durations[run] = duration
  255. def prepare(self, stdev=False):
  256. if len(self.durations) > 1:
  257. self.mean_duration = statistics.mean(self.durations)
  258. self.mean_count = statistics.mean(self.counts)
  259. if stdev:
  260. self.stdev_duration = statistics.stdev(self.durations)
  261. self.stdev_count = statistics.stdev(self.counts)
  262. def as_list(self):
  263. l = [self.name]
  264. for (c, d) in zip(self.counts, self.durations):
  265. l += [c, d]
  266. if self.mean_duration is not None:
  267. l += [self.mean_count]
  268. if self.stdev_count is not None:
  269. l += [self.stdev_count]
  270. l += [self.mean_duration]
  271. if self.stdev_duration is not None:
  272. l += [self.stdev_duration]
  273. return l
  274. def key(self):
  275. if self.mean_duration is not None:
  276. return self.mean_duration
  277. else:
  278. return self.durations[0]
  279. class Bucket:
  280. def __init__(self, name, run_count):
  281. self.name = name
  282. self.run_count = run_count
  283. self.data = {}
  284. self.table = None
  285. self.total_row = None
  286. def __repr__(self):
  287. s = "Bucket: " + self.name + " {\n"
  288. if self.table:
  289. s += "\n ".join(str(row) for row in self.table) + "\n"
  290. elif self.data:
  291. s += "\n ".join(str(row) for row in self.data.values()) + "\n"
  292. if self.total_row:
  293. s += " " + str(self.total_row) + "\n"
  294. return s + "}"
  295. def add_data_point(self, name, run, count, duration):
  296. if name not in self.data:
  297. self.data[name] = Row(name, self.run_count)
  298. self.data[name].add_data_point(run, count, duration)
  299. def prepare(self, stdev=False):
  300. if self.data:
  301. for row in self.data.values():
  302. row.prepare(stdev)
  303. self.table = sorted(self.data.values(), key=Row.key)
  304. self.total_row = Row("Total", self.run_count)
  305. self.total_row.add_data([
  306. sum(r.counts[i]
  307. for r in self.data.values())
  308. for i in range(0, self.run_count)
  309. ], [
  310. sum(r.durations[i]
  311. for r in self.data.values())
  312. for i in range(0, self.run_count)
  313. ])
  314. self.total_row.prepare(stdev)
  315. def as_list(self, add_bucket_titles=True, filter=None):
  316. t = []
  317. if filter is None or self.name in filter:
  318. if add_bucket_titles:
  319. t += [["\n"], [self.name]]
  320. t += [r.as_list() for r in self.table]
  321. t += [self.total_row.as_list()]
  322. return t
  323. def collect_buckets(story, group=True, repeats=1, output_dir="."):
  324. if group:
  325. groups = RUNTIME_CALL_STATS_GROUPS
  326. else:
  327. groups = []
  328. buckets = {}
  329. for i in range(0, repeats):
  330. story_dir = f"{story.replace(':', '_')}_{i + 1}"
  331. trace_dir = os.path.join(output_dir, "artifacts", story_dir, "trace",
  332. "traceEvents")
  333. # run_benchmark now dumps two files: a .pb.gz file and a .pb_converted.json
  334. # file. We only need the latter.
  335. trace_file_glob = os.path.join(trace_dir, "*" + JSON_FILE_EXTENSION)
  336. trace_files = glob.glob(trace_file_glob)
  337. if not trace_files:
  338. print("Could not find *%s file in %s" % (JSON_FILE_EXTENSION, trace_dir))
  339. sys.exit(1)
  340. if len(trace_files) > 1:
  341. print("Expecting one file but got: %s" % trace_files)
  342. sys.exit(1)
  343. trace_file = trace_files[0]
  344. output = process_trace(trace_file)
  345. for name in output:
  346. bucket_name = "Other"
  347. for group in groups:
  348. if group[1].match(name):
  349. bucket_name = group[0]
  350. break
  351. value = output[name]
  352. if bucket_name not in buckets:
  353. bucket = Bucket(bucket_name, repeats)
  354. buckets[bucket_name] = bucket
  355. else:
  356. bucket = buckets[bucket_name]
  357. bucket.add_data_point(name, i, value["count"], value["duration"] / 1000.0)
  358. return buckets
  359. def create_table(buckets, record_bucket_names=True, filter=None):
  360. table = []
  361. for bucket in buckets.values():
  362. table += bucket.as_list(
  363. add_bucket_titles=record_bucket_names, filter=filter)
  364. return table
  365. def main():
  366. args = parse_args()
  367. story = args.story[0]
  368. retain = args.retain
  369. if args.dir is not None:
  370. output_dir = args.dir
  371. if not os.path.isdir(output_dir):
  372. print("Specified output directory does not exist: " % output_dir)
  373. sys.exit(1)
  374. else:
  375. output_dir = tempfile.mkdtemp(prefix="runtime_call_stats_")
  376. run_benchmark(
  377. story,
  378. repeats=args.repeats,
  379. output_dir=output_dir,
  380. verbose=args.verbose,
  381. js_flags=args.js_flags,
  382. browser_args=args.browser_args,
  383. chromium_dir=args.chromium_dir,
  384. benchmark=args.benchmark,
  385. executable=args.executable,
  386. browser=args.browser,
  387. device=args.device)
  388. try:
  389. buckets = collect_buckets(
  390. story, group=args.group, repeats=args.repeats, output_dir=output_dir)
  391. for b in buckets.values():
  392. b.prepare(args.stdev)
  393. table = create_table(
  394. buckets, record_bucket_names=args.group, filter=args.filter)
  395. headers = [""] + ["Count", "Duration\n(ms)"] * args.repeats
  396. if args.repeats > 1:
  397. if args.stdev:
  398. headers += [
  399. "Count\nMean", "Count\nStdev", "Duration\nMean (ms)",
  400. "Duration\nStdev (ms)"
  401. ]
  402. else:
  403. headers += ["Count\nMean", "Duration\nMean (ms)"]
  404. if args.out_file:
  405. with open(args.out_file, "w", newline="") as f:
  406. write_output(f, table, headers, args.repeats, args.format)
  407. else:
  408. write_output(sys.stdout, table, headers, args.repeats, args.format)
  409. finally:
  410. if retain == "none":
  411. shutil.rmtree(output_dir)
  412. elif retain == "json":
  413. # Delete all files bottom up except ones ending in JSON_FILE_EXTENSION and
  414. # attempt to delete subdirectories (ignoring errors).
  415. for dir_name, subdir_list, file_list in os.walk(
  416. output_dir, topdown=False):
  417. for file_name in file_list:
  418. if not file_name.endswith(JSON_FILE_EXTENSION):
  419. os.remove(os.path.join(dir_name, file_name))
  420. for subdir in subdir_list:
  421. try:
  422. os.rmdir(os.path.join(dir_name, subdir))
  423. except OSError:
  424. pass
  425. if __name__ == "__main__":
  426. sys.exit(main())