avg.py 6.8 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248
  1. #!/usr/bin/env python
  2. # Copyright 2018 the V8 project authors. All rights reserved.
  3. # Use of this source code is governed by a BSD-style license that can
  4. # be found in the LICENSE file.
  5. """
  6. This script averages numbers output from another script. It is useful
  7. to average over a benchmark that outputs one or more results of the form
  8. <key> <number> <unit>
  9. key and unit are optional, but only one number per line is processed.
  10. For example, if
  11. $ bch --allow-natives-syntax toNumber.js
  12. outputs
  13. Number('undefined'): 155763
  14. (+'undefined'): 193050 Kps
  15. 23736 Kps
  16. then
  17. $ avg.py 10 bch --allow-natives-syntax toNumber.js
  18. will output
  19. [10/10] (+'undefined') : avg 192,240.40 stddev 6,486.24 (185,529.00 - 206,186.00)
  20. [10/10] Number('undefined') : avg 156,990.10 stddev 16,327.56 (144,718.00 - 202,840.00) Kps
  21. [10/10] [default] : avg 22,885.80 stddev 1,941.80 ( 17,584.00 - 24,266.00) Kps
  22. """
  23. # for py2/py3 compatibility
  24. from __future__ import print_function
  25. import argparse
  26. import math
  27. import re
  28. import signal
  29. import subprocess
  30. import sys
  31. PARSER = argparse.ArgumentParser(
  32. description="A script that averages numbers from another script's output",
  33. epilog="Example:\n\tavg.py 10 bash -c \"echo A: 100; echo B 120; sleep .1\""
  34. )
  35. PARSER.add_argument(
  36. 'repetitions',
  37. type=int,
  38. help="number of times the command should be repeated")
  39. PARSER.add_argument(
  40. 'command',
  41. nargs=argparse.REMAINDER,
  42. help="command to run (no quotes needed)")
  43. PARSER.add_argument(
  44. '--echo',
  45. '-e',
  46. action='store_true',
  47. default=False,
  48. help="set this flag to echo the command's output")
  49. ARGS = vars(PARSER.parse_args())
  50. if not ARGS['command']:
  51. print("No command provided.")
  52. exit(1)
  53. class FieldWidth:
  54. def __init__(self, points=0, key=0, average=0, stddev=0, min_width=0, max_width=0):
  55. self.widths = dict(points=points, key=key, average=average, stddev=stddev,
  56. min=min_width, max=max_width)
  57. def max_widths(self, other):
  58. self.widths = {k: max(v, other.widths[k]) for k, v in self.widths.items()}
  59. def __getattr__(self, key):
  60. return self.widths[key]
  61. def fmtS(string, width=0):
  62. return "{0:<{1}}".format(string, width)
  63. def fmtN(num, width=0):
  64. return "{0:>{1},.2f}".format(num, width)
  65. def fmt(num):
  66. return "{0:>,.2f}".format(num)
  67. def format_line(points, key, average, stddev, min_value, max_value,
  68. unit_string, widths):
  69. return "{:>{}}; {:<{}}; {:>{}}; {:>{}}; {:>{}}; {:>{}}; {}".format(
  70. points, widths.points,
  71. key, widths.key,
  72. average, widths.average,
  73. stddev, widths.stddev,
  74. min_value, widths.min,
  75. max_value, widths.max,
  76. unit_string)
  77. def fmt_reps(msrmnt):
  78. rep_string = str(ARGS['repetitions'])
  79. return "[{0:>{1}}/{2}]".format(msrmnt.size(), len(rep_string), rep_string)
  80. class Measurement:
  81. def __init__(self, key, unit):
  82. self.key = key
  83. self.unit = unit
  84. self.values = []
  85. self.average = 0
  86. self.count = 0
  87. self.M2 = 0
  88. self.min = float("inf")
  89. self.max = -float("inf")
  90. def addValue(self, value):
  91. try:
  92. num_value = float(value)
  93. self.values.append(num_value)
  94. self.min = min(self.min, num_value)
  95. self.max = max(self.max, num_value)
  96. self.count = self.count + 1
  97. delta = num_value - self.average
  98. self.average = self.average + delta / self.count
  99. delta2 = num_value - self.average
  100. self.M2 = self.M2 + delta * delta2
  101. except ValueError:
  102. print("Ignoring non-numeric value", value)
  103. def status(self, widths):
  104. return "{} {}: avg {} stddev {} ({} - {}) {}".format(
  105. fmt_reps(self),
  106. fmtS(self.key, widths.key), fmtN(self.average, widths.average),
  107. fmtN(self.stddev(), widths.stddev), fmtN(self.min, widths.min),
  108. fmtN(self.max, widths.max), fmtS(self.unit_string()))
  109. def result(self, widths):
  110. return format_line(self.size(), self.key, fmt(self.average),
  111. fmt(self.stddev()), fmt(self.min),
  112. fmt(self.max), self.unit_string(),
  113. widths)
  114. def unit_string(self):
  115. if not self.unit:
  116. return ""
  117. return self.unit
  118. def variance(self):
  119. if self.count < 2:
  120. return float('NaN')
  121. return self.M2 / (self.count - 1)
  122. def stddev(self):
  123. return math.sqrt(self.variance())
  124. def size(self):
  125. return len(self.values)
  126. def widths(self):
  127. return FieldWidth(
  128. points=len("{}".format(self.size())) + 2,
  129. key=len(self.key),
  130. average=len(fmt(self.average)),
  131. stddev=len(fmt(self.stddev())),
  132. min_width=len(fmt(self.min)),
  133. max_width=len(fmt(self.max)))
  134. def result_header(widths):
  135. return format_line("#/{}".format(ARGS['repetitions']),
  136. "id", "avg", "stddev", "min", "max", "unit", widths)
  137. class Measurements:
  138. def __init__(self):
  139. self.all = {}
  140. self.default_key = '[default]'
  141. self.max_widths = FieldWidth(
  142. points=len("{}".format(ARGS['repetitions'])) + 2,
  143. key=len("id"),
  144. average=len("avg"),
  145. stddev=len("stddev"),
  146. min_width=len("min"),
  147. max_width=len("max"))
  148. self.last_status_len = 0
  149. def record(self, key, value, unit):
  150. if not key:
  151. key = self.default_key
  152. if key not in self.all:
  153. self.all[key] = Measurement(key, unit)
  154. self.all[key].addValue(value)
  155. self.max_widths.max_widths(self.all[key].widths())
  156. def any(self):
  157. if self.all:
  158. return next(iter(self.all.values()))
  159. return None
  160. def print_results(self):
  161. print("{:<{}}".format("", self.last_status_len), end="\r")
  162. print(result_header(self.max_widths), sep=" ")
  163. for key in sorted(self.all):
  164. print(self.all[key].result(self.max_widths), sep=" ")
  165. def print_status(self):
  166. status = "No results found. Check format?"
  167. measurement = MEASUREMENTS.any()
  168. if measurement:
  169. status = measurement.status(MEASUREMENTS.max_widths)
  170. print("{:<{}}".format(status, self.last_status_len), end="\r")
  171. self.last_status_len = len(status)
  172. MEASUREMENTS = Measurements()
  173. def signal_handler(signum, frame):
  174. print("", end="\r")
  175. MEASUREMENTS.print_results()
  176. sys.exit(0)
  177. signal.signal(signal.SIGINT, signal_handler)
  178. SCORE_REGEX = (r'\A((console.timeEnd: )?'
  179. r'(?P<key>[^\s:,]+)[,:]?)?'
  180. r'(^\s*|\s+)'
  181. r'(?P<value>[0-9]+(.[0-9]+)?)'
  182. r'\ ?(?P<unit>[^\d\W]\w*)?[.\s]*\Z')
  183. for x in range(0, ARGS['repetitions']):
  184. proc = subprocess.Popen(ARGS['command'], stdout=subprocess.PIPE)
  185. for line in proc.stdout:
  186. if ARGS['echo']:
  187. print(line.decode(), end="")
  188. for m in re.finditer(SCORE_REGEX, line.decode()):
  189. MEASUREMENTS.record(m.group('key'), m.group('value'), m.group('unit'))
  190. proc.wait()
  191. if proc.returncode != 0:
  192. print("Child exited with status %d" % proc.returncode)
  193. break
  194. MEASUREMENTS.print_status()
  195. # Print final results
  196. MEASUREMENTS.print_results()