123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486 |
- #!/usr/bin/env python
- # Copyright 2017 the V8 project authors. All rights reserved.
- # Use of this source code is governed by a BSD-style license that can be
- # found in the LICENSE file.
- '''
- python %prog
- Compare perf trybot JSON files and output the results into a pleasing HTML page.
- Examples:
- %prog -t "ia32 results" Result,../result.json Master,/path-to/master.json -o results.html
- %prog -t "x64 results" ../result.json master.json -o results.html
- '''
- # for py2/py3 compatibility
- from __future__ import print_function
- from collections import OrderedDict
- import json
- import math
- from argparse import ArgumentParser
- import os
- import shutil
- import sys
- import tempfile
- PERCENT_CONSIDERED_SIGNIFICANT = 0.5
- PROBABILITY_CONSIDERED_SIGNIFICANT = 0.02
- PROBABILITY_CONSIDERED_MEANINGLESS = 0.05
- class Statistics:
- @staticmethod
- def Mean(values):
- return float(sum(values)) / len(values)
- @staticmethod
- def Variance(values, average):
- return map(lambda x: (x - average) ** 2, values)
- @staticmethod
- def StandardDeviation(values, average):
- return math.sqrt(Statistics.Mean(Statistics.Variance(values, average)))
- @staticmethod
- def ComputeZ(baseline_avg, baseline_sigma, mean, n):
- if baseline_sigma == 0:
- return 1000.0;
- return abs((mean - baseline_avg) / (baseline_sigma / math.sqrt(n)))
- # Values from http://www.fourmilab.ch/rpkp/experiments/analysis/zCalc.html
- @staticmethod
- def ComputeProbability(z):
- if z > 2.575829: # p 0.005: two sided < 0.01
- return 0
- if z > 2.326348: # p 0.010
- return 0.01
- if z > 2.170091: # p 0.015
- return 0.02
- if z > 2.053749: # p 0.020
- return 0.03
- if z > 1.959964: # p 0.025: two sided < 0.05
- return 0.04
- if z > 1.880793: # p 0.030
- return 0.05
- if z > 1.811910: # p 0.035
- return 0.06
- if z > 1.750686: # p 0.040
- return 0.07
- if z > 1.695397: # p 0.045
- return 0.08
- if z > 1.644853: # p 0.050: two sided < 0.10
- return 0.09
- if z > 1.281551: # p 0.100: two sided < 0.20
- return 0.10
- return 0.20 # two sided p >= 0.20
- class ResultsDiff:
- def __init__(self, significant, notable, percentage_string):
- self.significant_ = significant
- self.notable_ = notable
- self.percentage_string_ = percentage_string
- def percentage_string(self):
- return self.percentage_string_;
- def isSignificant(self):
- return self.significant_
- def isNotablyPositive(self):
- return self.notable_ > 0
- def isNotablyNegative(self):
- return self.notable_ < 0
- class BenchmarkResult:
- def __init__(self, units, count, result, sigma):
- self.units_ = units
- self.count_ = float(count)
- self.result_ = float(result)
- self.sigma_ = float(sigma)
- def Compare(self, other):
- if self.units_ != other.units_:
- print ("Incompatible units: %s and %s" % (self.units_, other.units_))
- sys.exit(1)
- significant = False
- notable = 0
- percentage_string = ""
- # compute notability and significance.
- if self.units_ == "score":
- compare_num = 100*self.result_/other.result_ - 100
- else:
- compare_num = 100*other.result_/self.result_ - 100
- if abs(compare_num) > 0.1:
- percentage_string = "%3.1f" % (compare_num)
- z = Statistics.ComputeZ(other.result_, other.sigma_,
- self.result_, self.count_)
- p = Statistics.ComputeProbability(z)
- if p < PROBABILITY_CONSIDERED_SIGNIFICANT:
- significant = True
- if compare_num >= PERCENT_CONSIDERED_SIGNIFICANT:
- notable = 1
- elif compare_num <= -PERCENT_CONSIDERED_SIGNIFICANT:
- notable = -1
- return ResultsDiff(significant, notable, percentage_string)
- def result(self):
- return self.result_
- def sigma(self):
- return self.sigma_
- class Benchmark:
- def __init__(self, name):
- self.name_ = name
- self.runs_ = {}
- def name(self):
- return self.name_
- def getResult(self, run_name):
- return self.runs_.get(run_name)
- def appendResult(self, run_name, trace):
- values = map(float, trace['results'])
- count = len(values)
- mean = Statistics.Mean(values)
- stddev = float(trace.get('stddev') or
- Statistics.StandardDeviation(values, mean))
- units = trace["units"]
- # print run_name, units, count, mean, stddev
- self.runs_[run_name] = BenchmarkResult(units, count, mean, stddev)
- class BenchmarkSuite:
- def __init__(self, name):
- self.name_ = name
- self.benchmarks_ = {}
- def SortedTestKeys(self):
- keys = self.benchmarks_.keys()
- keys.sort()
- t = "Total"
- if t in keys:
- keys.remove(t)
- keys.append(t)
- return keys
- def name(self):
- return self.name_
- def getBenchmark(self, benchmark_name):
- benchmark_object = self.benchmarks_.get(benchmark_name)
- if benchmark_object == None:
- benchmark_object = Benchmark(benchmark_name)
- self.benchmarks_[benchmark_name] = benchmark_object
- return benchmark_object
- class ResultTableRenderer:
- def __init__(self, output_file):
- self.benchmarks_ = []
- self.print_output_ = []
- self.output_file_ = output_file
- def Print(self, str_data):
- self.print_output_.append(str_data)
- def FlushOutput(self):
- string_data = "\n".join(self.print_output_)
- print_output = []
- if self.output_file_:
- # create a file
- with open(self.output_file_, "w") as text_file:
- text_file.write(string_data)
- else:
- print(string_data)
- def bold(self, data):
- return "<b>%s</b>" % data
- def red(self, data):
- return "<font color=\"red\">%s</font>" % data
- def green(self, data):
- return "<font color=\"green\">%s</font>" % data
- def PrintHeader(self):
- data = """<html>
- <head>
- <title>Output</title>
- <style type="text/css">
- /*
- Style inspired by Andy Ferra's gist at https://gist.github.com/andyferra/2554919
- */
- body {
- font-family: Helvetica, arial, sans-serif;
- font-size: 14px;
- line-height: 1.6;
- padding-top: 10px;
- padding-bottom: 10px;
- background-color: white;
- padding: 30px;
- }
- h1, h2, h3, h4, h5, h6 {
- margin: 20px 0 10px;
- padding: 0;
- font-weight: bold;
- -webkit-font-smoothing: antialiased;
- cursor: text;
- position: relative;
- }
- h1 {
- font-size: 28px;
- color: black;
- }
- h2 {
- font-size: 24px;
- border-bottom: 1px solid #cccccc;
- color: black;
- }
- h3 {
- font-size: 18px;
- }
- h4 {
- font-size: 16px;
- }
- h5 {
- font-size: 14px;
- }
- h6 {
- color: #777777;
- font-size: 14px;
- }
- p, blockquote, ul, ol, dl, li, table, pre {
- margin: 15px 0;
- }
- li p.first {
- display: inline-block;
- }
- ul, ol {
- padding-left: 30px;
- }
- ul :first-child, ol :first-child {
- margin-top: 0;
- }
- ul :last-child, ol :last-child {
- margin-bottom: 0;
- }
- table {
- padding: 0;
- }
- table tr {
- border-top: 1px solid #cccccc;
- background-color: white;
- margin: 0;
- padding: 0;
- }
- table tr:nth-child(2n) {
- background-color: #f8f8f8;
- }
- table tr th {
- font-weight: bold;
- border: 1px solid #cccccc;
- text-align: left;
- margin: 0;
- padding: 6px 13px;
- }
- table tr td {
- border: 1px solid #cccccc;
- text-align: right;
- margin: 0;
- padding: 6px 13px;
- }
- table tr td.name-column {
- text-align: left;
- }
- table tr th :first-child, table tr td :first-child {
- margin-top: 0;
- }
- table tr th :last-child, table tr td :last-child {
- margin-bottom: 0;
- }
- </style>
- </head>
- <body>
- """
- self.Print(data)
- def StartSuite(self, suite_name, run_names):
- self.Print("<h2>")
- self.Print("<a name=\"%s\">%s</a> <a href=\"#top\">(top)</a>" %
- (suite_name, suite_name))
- self.Print("</h2>");
- self.Print("<table class=\"benchmark\">")
- self.Print("<thead>")
- self.Print(" <th>Test</th>")
- main_run = None
- for run_name in run_names:
- self.Print(" <th>%s</th>" % run_name)
- if main_run == None:
- main_run = run_name
- else:
- self.Print(" <th>%</th>")
- self.Print("</thead>")
- self.Print("<tbody>")
- def FinishSuite(self):
- self.Print("</tbody>")
- self.Print("</table>")
- def StartBenchmark(self, benchmark_name):
- self.Print(" <tr>")
- self.Print(" <td class=\"name-column\">%s</td>" % benchmark_name)
- def FinishBenchmark(self):
- self.Print(" </tr>")
- def PrintResult(self, run):
- if run == None:
- self.PrintEmptyCell()
- return
- self.Print(" <td>%3.1f</td>" % run.result())
- def PrintComparison(self, run, main_run):
- if run == None or main_run == None:
- self.PrintEmptyCell()
- return
- diff = run.Compare(main_run)
- res = diff.percentage_string()
- if diff.isSignificant():
- res = self.bold(res)
- if diff.isNotablyPositive():
- res = self.green(res)
- elif diff.isNotablyNegative():
- res = self.red(res)
- self.Print(" <td>%s</td>" % res)
- def PrintEmptyCell(self):
- self.Print(" <td></td>")
- def StartTOC(self, title):
- self.Print("<h1>%s</h1>" % title)
- self.Print("<ul>")
- def FinishTOC(self):
- self.Print("</ul>")
- def PrintBenchmarkLink(self, benchmark):
- self.Print("<li><a href=\"#" + benchmark + "\">" + benchmark + "</a></li>")
- def PrintFooter(self):
- data = """</body>
- </html>
- """
- self.Print(data)
- def Render(args):
- benchmark_suites = {}
- run_names = OrderedDict()
- for json_file_list in args.json_file_list:
- run_name = json_file_list[0]
- if run_name.endswith(".json"):
- # The first item in the list is also a file name
- run_name = os.path.splitext(run_name)[0]
- filenames = json_file_list
- else:
- filenames = json_file_list[1:]
- for filename in filenames:
- print ("Processing result set \"%s\", file: %s" % (run_name, filename))
- with open(filename) as json_data:
- data = json.load(json_data)
- run_names[run_name] = 0
- for error in data["errors"]:
- print("Error:", error)
- for trace in data["traces"]:
- suite_name = trace["graphs"][0]
- benchmark_name = "/".join(trace["graphs"][1:])
- benchmark_suite_object = benchmark_suites.get(suite_name)
- if benchmark_suite_object == None:
- benchmark_suite_object = BenchmarkSuite(suite_name)
- benchmark_suites[suite_name] = benchmark_suite_object
- benchmark_object = benchmark_suite_object.getBenchmark(benchmark_name)
- benchmark_object.appendResult(run_name, trace);
- renderer = ResultTableRenderer(args.output)
- renderer.PrintHeader()
- title = args.title or "Benchmark results"
- renderer.StartTOC(title)
- for suite_name, benchmark_suite_object in sorted(benchmark_suites.iteritems()):
- renderer.PrintBenchmarkLink(suite_name)
- renderer.FinishTOC()
- for suite_name, benchmark_suite_object in sorted(benchmark_suites.iteritems()):
- renderer.StartSuite(suite_name, run_names)
- for benchmark_name in benchmark_suite_object.SortedTestKeys():
- benchmark_object = benchmark_suite_object.getBenchmark(benchmark_name)
- # print suite_name, benchmark_object.name()
- renderer.StartBenchmark(benchmark_name)
- main_run = None
- main_result = None
- for run_name in run_names:
- result = benchmark_object.getResult(run_name)
- renderer.PrintResult(result)
- if main_run == None:
- main_run = run_name
- main_result = result
- else:
- renderer.PrintComparison(result, main_result)
- renderer.FinishBenchmark()
- renderer.FinishSuite()
- renderer.PrintFooter()
- renderer.FlushOutput()
- def CommaSeparatedList(arg):
- return [x for x in arg.split(',')]
- if __name__ == '__main__':
- parser = ArgumentParser(description="Compare perf trybot JSON files and " +
- "output the results into a pleasing HTML page.")
- parser.add_argument("-t", "--title", dest="title",
- help="Optional title of the web page")
- parser.add_argument("-o", "--output", dest="output",
- help="Write html output to this file rather than stdout")
- parser.add_argument("json_file_list", nargs="+", type=CommaSeparatedList,
- help="[column name,]./path-to/result.json - a comma-separated" +
- " list of optional column name and paths to json files")
- args = parser.parse_args()
- Render(args)
|