perf-compare.py 12 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486
  1. #!/usr/bin/env python
  2. # Copyright 2017 the V8 project authors. All rights reserved.
  3. # Use of this source code is governed by a BSD-style license that can be
  4. # found in the LICENSE file.
  5. '''
  6. python %prog
  7. Compare perf trybot JSON files and output the results into a pleasing HTML page.
  8. Examples:
  9. %prog -t "ia32 results" Result,../result.json Master,/path-to/master.json -o results.html
  10. %prog -t "x64 results" ../result.json master.json -o results.html
  11. '''
  12. # for py2/py3 compatibility
  13. from __future__ import print_function
  14. from collections import OrderedDict
  15. import json
  16. import math
  17. from argparse import ArgumentParser
  18. import os
  19. import shutil
  20. import sys
  21. import tempfile
  22. PERCENT_CONSIDERED_SIGNIFICANT = 0.5
  23. PROBABILITY_CONSIDERED_SIGNIFICANT = 0.02
  24. PROBABILITY_CONSIDERED_MEANINGLESS = 0.05
  25. class Statistics:
  26. @staticmethod
  27. def Mean(values):
  28. return float(sum(values)) / len(values)
  29. @staticmethod
  30. def Variance(values, average):
  31. return map(lambda x: (x - average) ** 2, values)
  32. @staticmethod
  33. def StandardDeviation(values, average):
  34. return math.sqrt(Statistics.Mean(Statistics.Variance(values, average)))
  35. @staticmethod
  36. def ComputeZ(baseline_avg, baseline_sigma, mean, n):
  37. if baseline_sigma == 0:
  38. return 1000.0;
  39. return abs((mean - baseline_avg) / (baseline_sigma / math.sqrt(n)))
  40. # Values from http://www.fourmilab.ch/rpkp/experiments/analysis/zCalc.html
  41. @staticmethod
  42. def ComputeProbability(z):
  43. if z > 2.575829: # p 0.005: two sided < 0.01
  44. return 0
  45. if z > 2.326348: # p 0.010
  46. return 0.01
  47. if z > 2.170091: # p 0.015
  48. return 0.02
  49. if z > 2.053749: # p 0.020
  50. return 0.03
  51. if z > 1.959964: # p 0.025: two sided < 0.05
  52. return 0.04
  53. if z > 1.880793: # p 0.030
  54. return 0.05
  55. if z > 1.811910: # p 0.035
  56. return 0.06
  57. if z > 1.750686: # p 0.040
  58. return 0.07
  59. if z > 1.695397: # p 0.045
  60. return 0.08
  61. if z > 1.644853: # p 0.050: two sided < 0.10
  62. return 0.09
  63. if z > 1.281551: # p 0.100: two sided < 0.20
  64. return 0.10
  65. return 0.20 # two sided p >= 0.20
  66. class ResultsDiff:
  67. def __init__(self, significant, notable, percentage_string):
  68. self.significant_ = significant
  69. self.notable_ = notable
  70. self.percentage_string_ = percentage_string
  71. def percentage_string(self):
  72. return self.percentage_string_;
  73. def isSignificant(self):
  74. return self.significant_
  75. def isNotablyPositive(self):
  76. return self.notable_ > 0
  77. def isNotablyNegative(self):
  78. return self.notable_ < 0
  79. class BenchmarkResult:
  80. def __init__(self, units, count, result, sigma):
  81. self.units_ = units
  82. self.count_ = float(count)
  83. self.result_ = float(result)
  84. self.sigma_ = float(sigma)
  85. def Compare(self, other):
  86. if self.units_ != other.units_:
  87. print ("Incompatible units: %s and %s" % (self.units_, other.units_))
  88. sys.exit(1)
  89. significant = False
  90. notable = 0
  91. percentage_string = ""
  92. # compute notability and significance.
  93. if self.units_ == "score":
  94. compare_num = 100*self.result_/other.result_ - 100
  95. else:
  96. compare_num = 100*other.result_/self.result_ - 100
  97. if abs(compare_num) > 0.1:
  98. percentage_string = "%3.1f" % (compare_num)
  99. z = Statistics.ComputeZ(other.result_, other.sigma_,
  100. self.result_, self.count_)
  101. p = Statistics.ComputeProbability(z)
  102. if p < PROBABILITY_CONSIDERED_SIGNIFICANT:
  103. significant = True
  104. if compare_num >= PERCENT_CONSIDERED_SIGNIFICANT:
  105. notable = 1
  106. elif compare_num <= -PERCENT_CONSIDERED_SIGNIFICANT:
  107. notable = -1
  108. return ResultsDiff(significant, notable, percentage_string)
  109. def result(self):
  110. return self.result_
  111. def sigma(self):
  112. return self.sigma_
  113. class Benchmark:
  114. def __init__(self, name):
  115. self.name_ = name
  116. self.runs_ = {}
  117. def name(self):
  118. return self.name_
  119. def getResult(self, run_name):
  120. return self.runs_.get(run_name)
  121. def appendResult(self, run_name, trace):
  122. values = map(float, trace['results'])
  123. count = len(values)
  124. mean = Statistics.Mean(values)
  125. stddev = float(trace.get('stddev') or
  126. Statistics.StandardDeviation(values, mean))
  127. units = trace["units"]
  128. # print run_name, units, count, mean, stddev
  129. self.runs_[run_name] = BenchmarkResult(units, count, mean, stddev)
  130. class BenchmarkSuite:
  131. def __init__(self, name):
  132. self.name_ = name
  133. self.benchmarks_ = {}
  134. def SortedTestKeys(self):
  135. keys = self.benchmarks_.keys()
  136. keys.sort()
  137. t = "Total"
  138. if t in keys:
  139. keys.remove(t)
  140. keys.append(t)
  141. return keys
  142. def name(self):
  143. return self.name_
  144. def getBenchmark(self, benchmark_name):
  145. benchmark_object = self.benchmarks_.get(benchmark_name)
  146. if benchmark_object == None:
  147. benchmark_object = Benchmark(benchmark_name)
  148. self.benchmarks_[benchmark_name] = benchmark_object
  149. return benchmark_object
  150. class ResultTableRenderer:
  151. def __init__(self, output_file):
  152. self.benchmarks_ = []
  153. self.print_output_ = []
  154. self.output_file_ = output_file
  155. def Print(self, str_data):
  156. self.print_output_.append(str_data)
  157. def FlushOutput(self):
  158. string_data = "\n".join(self.print_output_)
  159. print_output = []
  160. if self.output_file_:
  161. # create a file
  162. with open(self.output_file_, "w") as text_file:
  163. text_file.write(string_data)
  164. else:
  165. print(string_data)
  166. def bold(self, data):
  167. return "<b>%s</b>" % data
  168. def red(self, data):
  169. return "<font color=\"red\">%s</font>" % data
  170. def green(self, data):
  171. return "<font color=\"green\">%s</font>" % data
  172. def PrintHeader(self):
  173. data = """<html>
  174. <head>
  175. <title>Output</title>
  176. <style type="text/css">
  177. /*
  178. Style inspired by Andy Ferra's gist at https://gist.github.com/andyferra/2554919
  179. */
  180. body {
  181. font-family: Helvetica, arial, sans-serif;
  182. font-size: 14px;
  183. line-height: 1.6;
  184. padding-top: 10px;
  185. padding-bottom: 10px;
  186. background-color: white;
  187. padding: 30px;
  188. }
  189. h1, h2, h3, h4, h5, h6 {
  190. margin: 20px 0 10px;
  191. padding: 0;
  192. font-weight: bold;
  193. -webkit-font-smoothing: antialiased;
  194. cursor: text;
  195. position: relative;
  196. }
  197. h1 {
  198. font-size: 28px;
  199. color: black;
  200. }
  201. h2 {
  202. font-size: 24px;
  203. border-bottom: 1px solid #cccccc;
  204. color: black;
  205. }
  206. h3 {
  207. font-size: 18px;
  208. }
  209. h4 {
  210. font-size: 16px;
  211. }
  212. h5 {
  213. font-size: 14px;
  214. }
  215. h6 {
  216. color: #777777;
  217. font-size: 14px;
  218. }
  219. p, blockquote, ul, ol, dl, li, table, pre {
  220. margin: 15px 0;
  221. }
  222. li p.first {
  223. display: inline-block;
  224. }
  225. ul, ol {
  226. padding-left: 30px;
  227. }
  228. ul :first-child, ol :first-child {
  229. margin-top: 0;
  230. }
  231. ul :last-child, ol :last-child {
  232. margin-bottom: 0;
  233. }
  234. table {
  235. padding: 0;
  236. }
  237. table tr {
  238. border-top: 1px solid #cccccc;
  239. background-color: white;
  240. margin: 0;
  241. padding: 0;
  242. }
  243. table tr:nth-child(2n) {
  244. background-color: #f8f8f8;
  245. }
  246. table tr th {
  247. font-weight: bold;
  248. border: 1px solid #cccccc;
  249. text-align: left;
  250. margin: 0;
  251. padding: 6px 13px;
  252. }
  253. table tr td {
  254. border: 1px solid #cccccc;
  255. text-align: right;
  256. margin: 0;
  257. padding: 6px 13px;
  258. }
  259. table tr td.name-column {
  260. text-align: left;
  261. }
  262. table tr th :first-child, table tr td :first-child {
  263. margin-top: 0;
  264. }
  265. table tr th :last-child, table tr td :last-child {
  266. margin-bottom: 0;
  267. }
  268. </style>
  269. </head>
  270. <body>
  271. """
  272. self.Print(data)
  273. def StartSuite(self, suite_name, run_names):
  274. self.Print("<h2>")
  275. self.Print("<a name=\"%s\">%s</a> <a href=\"#top\">(top)</a>" %
  276. (suite_name, suite_name))
  277. self.Print("</h2>");
  278. self.Print("<table class=\"benchmark\">")
  279. self.Print("<thead>")
  280. self.Print(" <th>Test</th>")
  281. main_run = None
  282. for run_name in run_names:
  283. self.Print(" <th>%s</th>" % run_name)
  284. if main_run == None:
  285. main_run = run_name
  286. else:
  287. self.Print(" <th>%</th>")
  288. self.Print("</thead>")
  289. self.Print("<tbody>")
  290. def FinishSuite(self):
  291. self.Print("</tbody>")
  292. self.Print("</table>")
  293. def StartBenchmark(self, benchmark_name):
  294. self.Print(" <tr>")
  295. self.Print(" <td class=\"name-column\">%s</td>" % benchmark_name)
  296. def FinishBenchmark(self):
  297. self.Print(" </tr>")
  298. def PrintResult(self, run):
  299. if run == None:
  300. self.PrintEmptyCell()
  301. return
  302. self.Print(" <td>%3.1f</td>" % run.result())
  303. def PrintComparison(self, run, main_run):
  304. if run == None or main_run == None:
  305. self.PrintEmptyCell()
  306. return
  307. diff = run.Compare(main_run)
  308. res = diff.percentage_string()
  309. if diff.isSignificant():
  310. res = self.bold(res)
  311. if diff.isNotablyPositive():
  312. res = self.green(res)
  313. elif diff.isNotablyNegative():
  314. res = self.red(res)
  315. self.Print(" <td>%s</td>" % res)
  316. def PrintEmptyCell(self):
  317. self.Print(" <td></td>")
  318. def StartTOC(self, title):
  319. self.Print("<h1>%s</h1>" % title)
  320. self.Print("<ul>")
  321. def FinishTOC(self):
  322. self.Print("</ul>")
  323. def PrintBenchmarkLink(self, benchmark):
  324. self.Print("<li><a href=\"#" + benchmark + "\">" + benchmark + "</a></li>")
  325. def PrintFooter(self):
  326. data = """</body>
  327. </html>
  328. """
  329. self.Print(data)
  330. def Render(args):
  331. benchmark_suites = {}
  332. run_names = OrderedDict()
  333. for json_file_list in args.json_file_list:
  334. run_name = json_file_list[0]
  335. if run_name.endswith(".json"):
  336. # The first item in the list is also a file name
  337. run_name = os.path.splitext(run_name)[0]
  338. filenames = json_file_list
  339. else:
  340. filenames = json_file_list[1:]
  341. for filename in filenames:
  342. print ("Processing result set \"%s\", file: %s" % (run_name, filename))
  343. with open(filename) as json_data:
  344. data = json.load(json_data)
  345. run_names[run_name] = 0
  346. for error in data["errors"]:
  347. print("Error:", error)
  348. for trace in data["traces"]:
  349. suite_name = trace["graphs"][0]
  350. benchmark_name = "/".join(trace["graphs"][1:])
  351. benchmark_suite_object = benchmark_suites.get(suite_name)
  352. if benchmark_suite_object == None:
  353. benchmark_suite_object = BenchmarkSuite(suite_name)
  354. benchmark_suites[suite_name] = benchmark_suite_object
  355. benchmark_object = benchmark_suite_object.getBenchmark(benchmark_name)
  356. benchmark_object.appendResult(run_name, trace);
  357. renderer = ResultTableRenderer(args.output)
  358. renderer.PrintHeader()
  359. title = args.title or "Benchmark results"
  360. renderer.StartTOC(title)
  361. for suite_name, benchmark_suite_object in sorted(benchmark_suites.iteritems()):
  362. renderer.PrintBenchmarkLink(suite_name)
  363. renderer.FinishTOC()
  364. for suite_name, benchmark_suite_object in sorted(benchmark_suites.iteritems()):
  365. renderer.StartSuite(suite_name, run_names)
  366. for benchmark_name in benchmark_suite_object.SortedTestKeys():
  367. benchmark_object = benchmark_suite_object.getBenchmark(benchmark_name)
  368. # print suite_name, benchmark_object.name()
  369. renderer.StartBenchmark(benchmark_name)
  370. main_run = None
  371. main_result = None
  372. for run_name in run_names:
  373. result = benchmark_object.getResult(run_name)
  374. renderer.PrintResult(result)
  375. if main_run == None:
  376. main_run = run_name
  377. main_result = result
  378. else:
  379. renderer.PrintComparison(result, main_result)
  380. renderer.FinishBenchmark()
  381. renderer.FinishSuite()
  382. renderer.PrintFooter()
  383. renderer.FlushOutput()
  384. def CommaSeparatedList(arg):
  385. return [x for x in arg.split(',')]
  386. if __name__ == '__main__':
  387. parser = ArgumentParser(description="Compare perf trybot JSON files and " +
  388. "output the results into a pleasing HTML page.")
  389. parser.add_argument("-t", "--title", dest="title",
  390. help="Optional title of the web page")
  391. parser.add_argument("-o", "--output", dest="output",
  392. help="Write html output to this file rather than stdout")
  393. parser.add_argument("json_file_list", nargs="+", type=CommaSeparatedList,
  394. help="[column name,]./path-to/result.json - a comma-separated" +
  395. " list of optional column name and paths to json files")
  396. args = parser.parse_args()
  397. Render(args)