test_results.py 9.5 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301
  1. # Copyright 2018 The Chromium Authors. All rights reserved.
  2. # Use of this source code is governed by a BSD-style license that can be
  3. # found in the LICENSE file.
  4. import json
  5. import logging
  6. import os
  7. from typing import Any, Dict, List, Tuple, Optional
  8. def UnitStringIsValid(unit: str) -> bool:
  9. """Checks to make sure that a given string is in fact a recognized unit used
  10. by the chromium perftests to report results.
  11. Args:
  12. unit (str): The unit string to be checked.
  13. Returns:
  14. bool: Whether or not it is a unit.
  15. """
  16. accepted_units = [
  17. "us/hop", "us/task", "ns/sample", "ms", "s", "count", "KB", "MB/s", "us"
  18. ]
  19. return unit in accepted_units
  20. class ResultLine(object):
  21. """ResultLine objects are each an individual line of output, complete with a
  22. unit, measurement, and descriptive component.
  23. """
  24. def __init__(self, desc: str, measure: float, unit: str) -> None:
  25. self.desc = desc
  26. self.meas = measure
  27. self.unit = unit
  28. def ToJsonDict(self) -> Dict[str, Any]:
  29. """Converts a ResultLine into a JSON-serializable dictionary object.
  30. Returns:
  31. Dict[str, Any]: A mapping of strings that will appear in the output JSON
  32. object to their respective values.
  33. """
  34. return {
  35. "description": self.desc,
  36. "measurement": self.meas,
  37. "unit": self.unit,
  38. }
  39. def ReadResultLineFromJson(dct: Dict[str, Any]) -> ResultLine:
  40. """Converts a JSON dictionary object into a ResultLine.
  41. Args:
  42. dct (Dict[str, Any]): The JSON object to be parsed as a ResultLine. MUST
  43. contain the strings 'description', 'measurement', and 'unit'.
  44. Raises:
  45. KeyError: If the passed in dictionary does not contain the three required
  46. strings that a serialized ResultLine must contain.
  47. Returns:
  48. ResultLine: A ResultLine object reconstituted from the JSON dictionary.
  49. """
  50. return ResultLine(dct["description"], float(dct["measurement"]), dct["unit"])
  51. def ResultLineFromStdout(line: str) -> Optional[ResultLine]:
  52. """Takes a line of stdout data and attempts to parse it into a ResultLine.
  53. Args:
  54. line (str): The stdout line to be converted
  55. Returns:
  56. Optional[ResultLine]: The output is Optional, because the line may be noise,
  57. or in some way incorrectly formatted and unparseable.
  58. """
  59. if "pkgsvr" in line:
  60. return None # Filters pkgsrv noise from Fuchsia output.
  61. chunks = line.split()
  62. # There should be 1 chunk for the measure, 1 for the unit, and at least one
  63. # for the line description, so at least 3 total
  64. if len(chunks) < 3:
  65. logging.warning("The line {} contains too few space-separated pieces to be "
  66. "parsed as a ResultLine".format(line))
  67. return None
  68. unit = chunks[-1]
  69. if not UnitStringIsValid(unit):
  70. logging.warning("The unit string parsed from {} was {}, which was not "
  71. "expected".format(line, unit))
  72. return None
  73. try:
  74. measure = float(chunks[-2])
  75. desc = " ".join(chunks[:-2])
  76. return ResultLine(desc, measure, unit)
  77. except ValueError as e:
  78. logging.warning("The chunk {} could not be parsed as a valid measurement "
  79. "because of {}".format(chunks[-2], str(e)))
  80. return None
  81. class TestResult(object):
  82. """TestResult objects comprise the smallest unit of a GTest target, and
  83. contain the name of the individual test run, and the time that the test took
  84. to run."""
  85. def __init__(self, name: str, time: float, lines: List[ResultLine]) -> None:
  86. self.name = name
  87. self.time = time
  88. self.lines = lines
  89. def ToJsonDict(self) -> Dict[str, Any]:
  90. """Converts a TestResult object to a JSON-serializable dictionary.
  91. Returns:
  92. Dict[str, Any]: The output dictionary object that can be directly
  93. serialized to JSON.
  94. """
  95. return {
  96. "name": self.name,
  97. "time_in_ms": self.time,
  98. "lines": [line.ToJsonDict() for line in self.lines]
  99. }
  100. def ReadTestFromJson(obj_dict: Dict[str, Any]) -> TestResult:
  101. """Reconstitutes a TestResult read from a JSON file back into a TestResult
  102. object.
  103. Args:
  104. obj_dict (Dict[str, Any]): The JSON object as read from an output JSON file.
  105. Returns:
  106. TestResult: The reconstituted TestResult object.
  107. """
  108. name = obj_dict["name"]
  109. time = obj_dict["time_in_ms"]
  110. lines = [ReadResultLineFromJson(line) for line in obj_dict["lines"]]
  111. return TestResult(name, time, lines)
  112. def ExtractTestInfo(line: str) -> Tuple[str, float]:
  113. """Deconstructs a line starting with OK, stating that the test finished
  114. successfully, and isolates the timing measurement as well as a descriptive
  115. string for the test
  116. Args:
  117. line (str): The line of output to attempt to destructure into name and time.
  118. Raises:
  119. Exception: In the event that it couldn't split on '(', because then it
  120. find the necessary timing measurement.
  121. Exception: in the event that it cannot find the ')' character in the output,
  122. because then it isn't capable of isolating the timing measurement.
  123. Returns:
  124. Tuple[str, float]: A tuple of the test name, and the amount of time it took
  125. to run.
  126. """
  127. # Trim off the [ OK ] part of the line
  128. trimmed = line.lstrip("[ OK ]").strip()
  129. try:
  130. test_name, rest = trimmed.split("(") # Isolate the measurement
  131. except Exception as e:
  132. err_text = "Could not extract the case name from {} because of error {}"\
  133. .format(trimmed, str(e))
  134. raise Exception(err_text)
  135. try:
  136. measure, _ = rest.split(")", 1)[0].split()
  137. except Exception as e:
  138. err_text = "Could not extract measure and units from {}\
  139. because of error {}".format(rest, str(e))
  140. raise Exception(err_text)
  141. return test_name.strip(), float(measure)
  142. def TaggedTestFromLines(lines: List[str]) -> TestResult:
  143. """Takes a chunk of lines gathered together from the stdout of a test process
  144. and collects it all into a single test result, including the set of
  145. ResultLines inside of the TestResult.
  146. Args:
  147. lines (List[str]): The stdout lines to be parsed into a single test result
  148. Returns:
  149. TestResult: The final parsed TestResult from the input
  150. """
  151. test_name, time = ExtractTestInfo(lines[-1])
  152. res_lines = []
  153. for line in lines[:-1]:
  154. res_line = ResultLineFromStdout(line)
  155. if res_line:
  156. res_lines.append(res_line)
  157. else:
  158. logging.warning("Couldn't parse line {} into a ResultLine".format(line))
  159. return TestResult(test_name, time, res_lines)
  160. class TargetResult(object):
  161. """TargetResult objects contain the entire set of TestSuite objects that are
  162. invoked by a single test target, such as base:base_perftests and the like.
  163. They also include the name of the target, and the time it took the target to
  164. run.
  165. """
  166. def __init__(self, name: str, tests: List[TestResult]) -> None:
  167. self.name = name
  168. self.tests = tests
  169. def ToJsonDict(self) -> Dict[str, Any]:
  170. """Converts a TargetResult to a JSON-serializable dict.
  171. Returns:
  172. Dict[str, Any]: The TargetResult in JSON-serializable form.
  173. """
  174. return {
  175. "name": self.name,
  176. "tests": [test.ToJsonDict() for test in self.tests]
  177. }
  178. def WriteToJson(self, path: str) -> None:
  179. """Writes this TargetResult object as a JSON file at the given file path.
  180. Args:
  181. path (str): The location to place the serialized version of this object.
  182. Returns:
  183. None: It'd return IO (), but this Python.
  184. """
  185. with open(path, "w") as outfile:
  186. json.dump(self.ToJsonDict(), outfile, indent=2)
  187. def ReadTargetFromJson(path: str) -> Optional[TargetResult]:
  188. """Takes a file path and attempts to parse a TargetResult from it.
  189. Args:
  190. path (str): The location of the JSON-serialized TargetResult to read.
  191. Returns:
  192. Optional[TargetResult]: Again, technically should be wrapped in an IO,
  193. but Python.
  194. """
  195. with open(path, "r") as json_file:
  196. dct = json.load(json_file)
  197. return TargetResult(
  198. dct["name"], [ReadTestFromJson(test_dct) for test_dct in dct["tests"]])
  199. def TargetResultFromStdout(lines: List[str], name: str) -> TargetResult:
  200. """TargetResultFromStdout attempts to associate GTest names to the lines of
  201. output that they produce. Example input looks something like the following:
  202. [ RUN ] TestNameFoo
  203. INFO measurement units
  204. ...
  205. [ OK ] TestNameFoo (measurement units)
  206. ...
  207. Unfortunately, Because the results of output from perftest targets is not
  208. necessarily consistent between test targets, this makes a best-effort to parse
  209. as much information from them as possible.
  210. Args:
  211. lines (List[str]): The entire list of lines from the standard output to be
  212. parsed.
  213. name (str): The name of the Target that generated the output. Necessary to
  214. be able to give the TargetResult a meaningful name.
  215. Returns:
  216. TargetResult: The TargetResult object parsed from the input lines.
  217. """
  218. test_line_lists = [] # type: List[List[str]]
  219. test_line_accum = [] # type: List[str]
  220. read_lines = False
  221. for line in lines:
  222. # We're starting a test suite
  223. if line.startswith("[ RUN ]"):
  224. read_lines = True
  225. # We have a prior suite that needs to be added
  226. if len(test_line_accum) > 0:
  227. test_line_lists.append(test_line_accum)
  228. test_line_accum = []
  229. elif read_lines:
  230. # We don't actually care about the data in the RUN line, just its
  231. # presence. the OK line contains the same info, as well as the total test
  232. # run time
  233. test_line_accum.append(line)
  234. if line.startswith("[ OK ]"):
  235. read_lines = False
  236. test_cases = [
  237. TaggedTestFromLines(test_lines) for test_lines in test_line_lists
  238. ]
  239. return TargetResult(name, test_cases)