generate_perf_report.py 13 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402
  1. #!/usr/bin/env python3
  2. # Copyright 2018 The Chromium Authors. All rights reserved.
  3. # Use of this source code is governed by a BSD-style license that can be
  4. # found in the LICENSE file.
  5. """generate_perf_report.py is to be used after comparative_tester.py has been
  6. executed and written some test data into the location specified by
  7. target_spec.py. It writes to results_dir and reads all present test info from
  8. raw_data_dir. Using this script should just be a matter of invoking it from
  9. chromium/src while raw test data exists in raw_data_dir."""
  10. import json
  11. import logging
  12. import math
  13. import os
  14. import sys
  15. from typing import List, Dict, Set, Tuple, Optional, Any, TypeVar, Callable
  16. import target_spec
  17. from test_results import (TargetResult, ReadTargetFromJson, TestResult,
  18. ResultLine)
  19. class LineStats(object):
  20. def __init__(self, desc: str, unit: str, time_avg: float, time_dev: float,
  21. cv: float, samples: int) -> None:
  22. """A corpus of stats about a particular line from a given test's output.
  23. Args:
  24. desc (str): Descriptive text of the line in question.
  25. unit (str): The units of measure that the line's result is in.
  26. time_avg (float): The average measurement.
  27. time_dev (float): The standard deviation of the measurement.
  28. cv (float): The coefficient of variance of the measure.
  29. samples (int): The number of samples that went into making this object.
  30. """
  31. self.desc = desc
  32. self.time_avg = time_avg
  33. self.time_dev = time_dev
  34. self.cv = cv
  35. self.unit = unit
  36. self.sample_num = samples
  37. def ToString(self) -> str:
  38. """Converts the line to a human-readable string."""
  39. if self.sample_num > 1:
  40. return "{}: {:.5f} σ={:.5f} {} with n={} cv={}".format(
  41. self.desc, self.time_avg, self.time_dev, self.unit, self.sample_num,
  42. self.cv)
  43. else:
  44. return "{}: {:.5f} with only one sample".format(self.desc, self.time_avg)
  45. def LineFromList(lines: List[ResultLine]) -> LineStats:
  46. """Takes a list of ResultLines and generates statistics for them.
  47. Args:
  48. lines (List[ResultLine]): The list of lines to generate stats for.
  49. Returns:
  50. LineStats: the representation of statistical data for the lines.
  51. """
  52. desc = lines[0].desc
  53. unit = lines[0].unit
  54. times = [line.meas for line in lines]
  55. avg, dev, cv = GenStats(times)
  56. return LineStats(desc, unit, avg, dev, cv, len(lines))
  57. class TestStats(object):
  58. def __init__(self, name: str, time_avg: float, time_dev: float, cv: float,
  59. samples: int, lines: List[LineStats]) -> None:
  60. """Represents a summary of relevant statistics for a list of tests.
  61. Args:
  62. name (str): The name of the test whose runs are being averaged.
  63. time_avg (float): The average time to execute the test.
  64. time_dev (float): The standard deviation in the mean.
  65. cv (float): The coefficient of variance of the population.
  66. samples (int): The number of samples in the population
  67. lines (List[LineStats]): The averaged list of all the lines of output that
  68. comprises this test.
  69. """
  70. self.name = name
  71. self.time_avg = time_avg
  72. self.time_dev = time_dev
  73. self.cv = cv
  74. self.sample_num = samples
  75. self.lines = lines
  76. def ToLines(self) -> List[str]:
  77. """The stats of this test, as well as its constituent LineStats, in a human-
  78. readable format.
  79. Returns:
  80. List[str]: The human-readable list of lines.
  81. """
  82. lines = []
  83. if self.sample_num > 1:
  84. lines.append("{}: {:.5f} σ={:.5f}ms with n={} cv={}".format(
  85. self.name, self.time_avg, self.time_dev, self.sample_num, self.cv))
  86. else:
  87. lines.append("{}: {:.5f} with only one sample".format(
  88. self.name, self.time_avg))
  89. for line in self.lines:
  90. lines.append(" {}".format(line.ToString()))
  91. return lines
  92. def TestFromList(tests: List[TestResult]) -> TestStats:
  93. """Coalesces a list of TestResults into a single TestStats object.
  94. Args:
  95. tests (List[TestResult]): The input sample of the tests.
  96. Returns:
  97. TestStats: A representation of the statistics of the tests.
  98. """
  99. name = tests[0].name
  100. avg, dev, cv = GenStats([test.time for test in tests])
  101. lines = {} # type: Dict[str, List[ResultLine]]
  102. for test in tests:
  103. assert test.name == name
  104. for line in test.lines:
  105. if not line.desc in lines:
  106. lines[line.desc] = [line]
  107. else:
  108. lines[line.desc].append(line)
  109. test_lines = []
  110. for _, line_list in lines.items():
  111. stat_line = LineFromList(line_list)
  112. if stat_line:
  113. test_lines.append(stat_line)
  114. return TestStats(name, avg, dev, cv, len(tests), test_lines)
  115. class TargetStats(object):
  116. def __init__(self, name: str, samples: int, tests: List[TestStats]) -> None:
  117. """A representation of the actual target that was built and run on the
  118. platforms multiple times to generate statistical data.
  119. Args:
  120. name (str): The name of the target that was built and run.
  121. samples (int): The number of times the tests were run.
  122. tests (List[TestStats]): The statistics of tests included in the target.
  123. """
  124. self.name = name
  125. self.sample_num = samples
  126. self.tests = tests
  127. def ToLines(self) -> List[str]:
  128. """Converts the entire target into a list of lines in human-readable format.
  129. Returns:
  130. List[str]: The human-readable test lines.
  131. """
  132. lines = []
  133. if self.sample_num > 1:
  134. lines.append("{}: ".format(self.name))
  135. else:
  136. lines.append("{}: with only one sample".format(self.name))
  137. for test in self.tests:
  138. for line in test.ToLines():
  139. lines.append(" {}".format(line))
  140. return lines
  141. def __format__(self, format_spec):
  142. return "\n".join(self.ToLines())
  143. def TargetFromList(results: List[TargetResult]) -> TargetStats:
  144. """Coalesces a list of TargetResults into a single collection of stats.
  145. Args:
  146. results (List[TargetResult]): The sampling of target executions to generate
  147. stats for.
  148. Returns:
  149. TargetStats: The body of stats for the sample given.
  150. """
  151. name = results[0].name
  152. sample_num = len(results)
  153. tests = {} # type: Dict[str, List[TestResult]]
  154. for result in results:
  155. assert result.name == name
  156. # This groups tests by name so that they can be considered independently,
  157. # so that in the event tests flake out, their average times can
  158. # still be accurately calculated
  159. for test in result.tests:
  160. if not test.name in tests.keys():
  161. tests[test.name] = [test]
  162. tests[test.name].append(test)
  163. test_stats = [TestFromList(test_list) for _, test_list in tests.items()]
  164. return TargetStats(name, sample_num, test_stats)
  165. def GenStats(corpus: List[float]) -> Tuple[float, float, float]:
  166. """Generates statistics from a list of values
  167. Args:
  168. corpus (List[float]): The set of data to generate statistics for.
  169. Returns:
  170. Tuple[float, float, float]: The mean, standard deviation, and coefficient of
  171. variation for the given sample data.
  172. """
  173. avg = sum(corpus) / len(corpus)
  174. adjusted_sum = 0.0
  175. for item in corpus:
  176. adjusted = item - avg
  177. adjusted_sum += adjusted * adjusted
  178. dev = math.sqrt(adjusted_sum / len(corpus))
  179. cv = dev / avg
  180. return avg, dev, cv
  181. def DirectoryStats(directory: str) -> List[TargetStats]:
  182. """Takes a path to directory, and uses JSON files in that directory to compile
  183. a list of statistical objects for each independent test target it can detect
  184. in the directory.
  185. Args:
  186. directory (str): The directory to scan for relevant JSONs
  187. Returns:
  188. List[TargetStats]: Each element in this list is one target, averaged up over
  189. all of its executions.
  190. """
  191. resultMap = {} # type: Dict[str, List[TargetResult]]
  192. for file in os.listdir(directory):
  193. results = ReadTargetFromJson("{}/{}".format(directory, file))
  194. if not results.name in resultMap.keys():
  195. resultMap[results.name] = [results]
  196. else:
  197. resultMap[results.name].append(results)
  198. targets = []
  199. for _, resultList in resultMap.items():
  200. targets.append(TargetFromList(resultList))
  201. return targets
  202. def CompareTargets(linux: TargetStats, fuchsia: TargetStats) -> Dict[str, Any]:
  203. """Compare takes a corpus of statistics from both Fuchsia and Linux, and then
  204. lines up the values, compares them to each other, and writes them into a
  205. dictionary that can be JSONified.
  206. """
  207. if linux and fuchsia:
  208. assert linux.name == fuchsia.name
  209. paired_tests = ZipListsByPredicate(linux.tests, fuchsia.tests,
  210. lambda test: test.name)
  211. paired_tests = MapDictValues(paired_tests, CompareTests)
  212. return {"name": linux.name, "tests": paired_tests}
  213. else:
  214. # One of them has to be non-null, by the way ZipListsByPredicate functions
  215. assert linux or fuchsia
  216. if linux:
  217. logging.error("Fuchsia was missing test target {}".format(linux.name))
  218. else:
  219. logging.error("Linux was missing test target {}".format(fuchsia.name))
  220. return None
  221. def CompareTests(linux: TestStats, fuchsia: TestStats) -> Dict[str, Any]:
  222. """As CompareTargets, but at the test level"""
  223. if not linux and not fuchsia:
  224. logging.error("Two null TestStats objects were passed to CompareTests.")
  225. return {}
  226. if not linux or not fuchsia:
  227. if linux:
  228. name = linux.name
  229. failing_os = "Fuchsia"
  230. else:
  231. name = fuchsia.name
  232. failing_os = "Linux"
  233. logging.error("%s failed to produce output for the test %s",
  234. failing_os, name)
  235. return {}
  236. assert linux.name == fuchsia.name
  237. paired_lines = ZipListsByPredicate(linux.lines, fuchsia.lines,
  238. lambda line: line.desc)
  239. paired_lines = MapDictValues(paired_lines, CompareLines)
  240. result = {"lines": paired_lines, "unit": "ms"} # type: Dict[str, Any]
  241. if linux:
  242. result["name"] = linux.name
  243. result["linux_avg"] = linux.time_avg
  244. result["linux_dev"] = linux.time_dev
  245. result["linux_cv"] = linux.cv
  246. if fuchsia == None:
  247. logging.warning("Fuchsia is missing test case {}".format(linux.name))
  248. else:
  249. result["name"] = fuchsia.name
  250. result["fuchsia_avg"] = fuchsia.time_avg
  251. result["fuchsia_dev"] = fuchsia.time_dev
  252. result["fuchsia_cv"] = fuchsia.cv
  253. return result
  254. def CompareLines(linux: LineStats, fuchsia: LineStats) -> Dict[str, Any]:
  255. """CompareLines wraps two LineStats objects up as a JSON-dumpable dict.
  256. It also logs a warning every time a line is given which can't be matched up.
  257. If both lines passed are None, or their units or descriptions are not the same
  258. (which should never happen) this function fails.
  259. """
  260. if linux != None and fuchsia != None:
  261. assert linux.desc == fuchsia.desc
  262. assert linux.unit == fuchsia.unit
  263. assert linux != None or fuchsia != None
  264. # ref_test is because we don't actually care which test we get the values
  265. # from, as long as we get values for the name and description
  266. ref_test = linux if linux else fuchsia
  267. result = {"desc": ref_test.desc, "unit": ref_test.unit}
  268. if fuchsia == None:
  269. logging.warning("Fuchsia is missing test line {}".format(linux.desc))
  270. else:
  271. result["fuchsia_avg"] = fuchsia.time_avg
  272. result["fuchsia_dev"] = fuchsia.time_dev
  273. result["fuchsia_cv"] = fuchsia.cv
  274. if linux:
  275. result["linux_avg"] = linux.time_avg
  276. result["linux_dev"] = linux.time_dev
  277. result["linux_cv"] = linux.cv
  278. return result
  279. T = TypeVar("T")
  280. R = TypeVar("R")
  281. def ZipListsByPredicate(left: List[T], right: List[T],
  282. pred: Callable[[T], R]) -> Dict[R, Tuple[T, T]]:
  283. """This function takes two lists, and a predicate. The predicate is applied to
  284. the values in both lists to obtain a keying value from them. Each item is then
  285. inserted into the returned dictionary using the obtained key. The predicate
  286. should not map multiple values from one list to the same key.
  287. """
  288. paired_items = {} # type: Dict [R, Tuple[T, T]]
  289. for item in left:
  290. key = pred(item)
  291. # the first list shouldn't cause any key collisions
  292. assert key not in paired_items.keys()
  293. paired_items[key] = item, None
  294. for item in right:
  295. key = pred(item)
  296. if key in paired_items.keys():
  297. # elem 1 of the tuple is always None if the key exists in the map
  298. prev, _ = paired_items[key]
  299. paired_items[key] = prev, item
  300. else:
  301. paired_items[key] = None, item
  302. return paired_items
  303. U = TypeVar("U")
  304. V = TypeVar("V")
  305. def MapDictValues(dct: Dict[T, Tuple[R, U]],
  306. predicate: Callable[[R, U], V]) -> Dict[T, V]:
  307. """This function applies the predicate to all the values in the dictionary,
  308. returning a new dictionary with the new values.
  309. """
  310. out_dict = {}
  311. for key, val in dct.items():
  312. out_dict[key] = predicate(*val)
  313. return out_dict
  314. def main():
  315. linux_avgs = DirectoryStats(target_spec.raw_linux_dir)
  316. fuchsia_avgs = DirectoryStats(target_spec.raw_fuchsia_dir)
  317. paired_targets = ZipListsByPredicate(linux_avgs, fuchsia_avgs,
  318. lambda target: target.name)
  319. for name, targets in paired_targets.items():
  320. comparison_dict = CompareTargets(*targets)
  321. if comparison_dict:
  322. with open("{}/{}.json".format(target_spec.results_dir, name),
  323. "w") as outfile:
  324. json.dump(comparison_dict, outfile, indent=2)
  325. if __name__ == "__main__":
  326. sys.exit(main())