buildstats.py 12 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346
  1. #
  2. # Copyright (c) 2017, Intel Corporation.
  3. #
  4. # SPDX-License-Identifier: GPL-2.0-only
  5. #
  6. """Functionality for analyzing buildstats"""
  7. import json
  8. import logging
  9. import os
  10. import re
  11. from collections import namedtuple,OrderedDict
  12. from statistics import mean
  13. log = logging.getLogger()
  14. taskdiff_fields = ('pkg', 'pkg_op', 'task', 'task_op', 'value1', 'value2',
  15. 'absdiff', 'reldiff')
  16. TaskDiff = namedtuple('TaskDiff', ' '.join(taskdiff_fields))
  17. class BSError(Exception):
  18. """Error handling of buildstats"""
  19. pass
  20. class BSTask(dict):
  21. def __init__(self, *args, **kwargs):
  22. self['start_time'] = None
  23. self['elapsed_time'] = None
  24. self['status'] = None
  25. self['iostat'] = {}
  26. self['rusage'] = {}
  27. self['child_rusage'] = {}
  28. super(BSTask, self).__init__(*args, **kwargs)
  29. @property
  30. def cputime(self):
  31. """Sum of user and system time taken by the task"""
  32. rusage = self['rusage']['ru_stime'] + self['rusage']['ru_utime']
  33. if self['child_rusage']:
  34. # Child rusage may have been optimized out
  35. return rusage + self['child_rusage']['ru_stime'] + self['child_rusage']['ru_utime']
  36. else:
  37. return rusage
  38. @property
  39. def walltime(self):
  40. """Elapsed wall clock time"""
  41. return self['elapsed_time']
  42. @property
  43. def read_bytes(self):
  44. """Bytes read from the block layer"""
  45. return self['iostat']['read_bytes']
  46. @property
  47. def write_bytes(self):
  48. """Bytes written to the block layer"""
  49. return self['iostat']['write_bytes']
  50. @property
  51. def read_ops(self):
  52. """Number of read operations on the block layer"""
  53. if self['child_rusage']:
  54. # Child rusage may have been optimized out
  55. return self['rusage']['ru_inblock'] + self['child_rusage']['ru_inblock']
  56. else:
  57. return self['rusage']['ru_inblock']
  58. @property
  59. def write_ops(self):
  60. """Number of write operations on the block layer"""
  61. if self['child_rusage']:
  62. # Child rusage may have been optimized out
  63. return self['rusage']['ru_oublock'] + self['child_rusage']['ru_oublock']
  64. else:
  65. return self['rusage']['ru_oublock']
  66. @classmethod
  67. def from_file(cls, buildstat_file):
  68. """Read buildstat text file"""
  69. bs_task = cls()
  70. log.debug("Reading task buildstats from %s", buildstat_file)
  71. end_time = None
  72. with open(buildstat_file) as fobj:
  73. for line in fobj.readlines():
  74. key, val = line.split(':', 1)
  75. val = val.strip()
  76. if key == 'Started':
  77. start_time = float(val)
  78. bs_task['start_time'] = start_time
  79. elif key == 'Ended':
  80. end_time = float(val)
  81. elif key.startswith('IO '):
  82. split = key.split()
  83. bs_task['iostat'][split[1]] = int(val)
  84. elif key.find('rusage') >= 0:
  85. split = key.split()
  86. ru_key = split[-1]
  87. if ru_key in ('ru_stime', 'ru_utime'):
  88. val = float(val)
  89. else:
  90. val = int(val)
  91. ru_type = 'rusage' if split[0] == 'rusage' else \
  92. 'child_rusage'
  93. bs_task[ru_type][ru_key] = val
  94. elif key == 'Status':
  95. bs_task['status'] = val
  96. if end_time is not None and start_time is not None:
  97. bs_task['elapsed_time'] = end_time - start_time
  98. else:
  99. raise BSError("{} looks like a invalid buildstats file".format(buildstat_file))
  100. return bs_task
  101. class BSTaskAggregate(object):
  102. """Class representing multiple runs of the same task"""
  103. properties = ('cputime', 'walltime', 'read_bytes', 'write_bytes',
  104. 'read_ops', 'write_ops')
  105. def __init__(self, tasks=None):
  106. self._tasks = tasks or []
  107. self._properties = {}
  108. def __getattr__(self, name):
  109. if name in self.properties:
  110. if name not in self._properties:
  111. # Calculate properties on demand only. We only provide mean
  112. # value, so far
  113. self._properties[name] = mean([getattr(t, name) for t in self._tasks])
  114. return self._properties[name]
  115. else:
  116. raise AttributeError("'BSTaskAggregate' has no attribute '{}'".format(name))
  117. def append(self, task):
  118. """Append new task"""
  119. # Reset pre-calculated properties
  120. assert isinstance(task, BSTask), "Type is '{}' instead of 'BSTask'".format(type(task))
  121. self._properties = {}
  122. self._tasks.append(task)
  123. class BSRecipe(object):
  124. """Class representing buildstats of one recipe"""
  125. def __init__(self, name, epoch, version, revision):
  126. self.name = name
  127. self.epoch = epoch
  128. self.version = version
  129. self.revision = revision
  130. if epoch is None:
  131. self.evr = "{}-{}".format(version, revision)
  132. else:
  133. self.evr = "{}_{}-{}".format(epoch, version, revision)
  134. self.tasks = {}
  135. def aggregate(self, bsrecipe):
  136. """Aggregate data of another recipe buildstats"""
  137. if self.nevr != bsrecipe.nevr:
  138. raise ValueError("Refusing to aggregate buildstats, recipe version "
  139. "differs: {} vs. {}".format(self.nevr, bsrecipe.nevr))
  140. if set(self.tasks.keys()) != set(bsrecipe.tasks.keys()):
  141. raise ValueError("Refusing to aggregate buildstats, set of tasks "
  142. "in {} differ".format(self.name))
  143. for taskname, taskdata in bsrecipe.tasks.items():
  144. if not isinstance(self.tasks[taskname], BSTaskAggregate):
  145. self.tasks[taskname] = BSTaskAggregate([self.tasks[taskname]])
  146. self.tasks[taskname].append(taskdata)
  147. @property
  148. def nevr(self):
  149. return self.name + '-' + self.evr
  150. class BuildStats(dict):
  151. """Class representing buildstats of one build"""
  152. @property
  153. def num_tasks(self):
  154. """Get number of tasks"""
  155. num = 0
  156. for recipe in self.values():
  157. num += len(recipe.tasks)
  158. return num
  159. @classmethod
  160. def from_json(cls, bs_json):
  161. """Create new BuildStats object from JSON object"""
  162. buildstats = cls()
  163. for recipe in bs_json:
  164. if recipe['name'] in buildstats:
  165. raise BSError("Cannot handle multiple versions of the same "
  166. "package ({})".format(recipe['name']))
  167. bsrecipe = BSRecipe(recipe['name'], recipe['epoch'],
  168. recipe['version'], recipe['revision'])
  169. for task, data in recipe['tasks'].items():
  170. bsrecipe.tasks[task] = BSTask(data)
  171. buildstats[recipe['name']] = bsrecipe
  172. return buildstats
  173. @staticmethod
  174. def from_file_json(path):
  175. """Load buildstats from a JSON file"""
  176. with open(path) as fobj:
  177. bs_json = json.load(fobj)
  178. return BuildStats.from_json(bs_json)
  179. @staticmethod
  180. def split_nevr(nevr):
  181. """Split name and version information from recipe "nevr" string"""
  182. n_e_v, revision = nevr.rsplit('-', 1)
  183. match = re.match(r'^(?P<name>\S+)-((?P<epoch>[0-9]{1,5})_)?(?P<version>[0-9]\S*)$',
  184. n_e_v)
  185. if not match:
  186. # If we're not able to parse a version starting with a number, just
  187. # take the part after last dash
  188. match = re.match(r'^(?P<name>\S+)-((?P<epoch>[0-9]{1,5})_)?(?P<version>[^-]+)$',
  189. n_e_v)
  190. name = match.group('name')
  191. version = match.group('version')
  192. epoch = match.group('epoch')
  193. return name, epoch, version, revision
  194. @classmethod
  195. def from_dir(cls, path):
  196. """Load buildstats from a buildstats directory"""
  197. if not os.path.isfile(os.path.join(path, 'build_stats')):
  198. raise BSError("{} does not look like a buildstats directory".format(path))
  199. log.debug("Reading buildstats directory %s", path)
  200. buildstats = cls()
  201. subdirs = os.listdir(path)
  202. for dirname in subdirs:
  203. recipe_dir = os.path.join(path, dirname)
  204. if not os.path.isdir(recipe_dir):
  205. continue
  206. name, epoch, version, revision = cls.split_nevr(dirname)
  207. bsrecipe = BSRecipe(name, epoch, version, revision)
  208. for task in os.listdir(recipe_dir):
  209. bsrecipe.tasks[task] = BSTask.from_file(
  210. os.path.join(recipe_dir, task))
  211. if name in buildstats:
  212. raise BSError("Cannot handle multiple versions of the same "
  213. "package ({})".format(name))
  214. buildstats[name] = bsrecipe
  215. return buildstats
  216. def aggregate(self, buildstats):
  217. """Aggregate other buildstats into this"""
  218. if set(self.keys()) != set(buildstats.keys()):
  219. raise ValueError("Refusing to aggregate buildstats, set of "
  220. "recipes is different: %s" % (set(self.keys()) ^ set(buildstats.keys())))
  221. for pkg, data in buildstats.items():
  222. self[pkg].aggregate(data)
  223. def diff_buildstats(bs1, bs2, stat_attr, min_val=None, min_absdiff=None, only_tasks=[]):
  224. """Compare the tasks of two buildstats"""
  225. tasks_diff = []
  226. pkgs = set(bs1.keys()).union(set(bs2.keys()))
  227. for pkg in pkgs:
  228. tasks1 = bs1[pkg].tasks if pkg in bs1 else {}
  229. tasks2 = bs2[pkg].tasks if pkg in bs2 else {}
  230. if only_tasks:
  231. tasks1 = {k: v for k, v in tasks1.items() if k in only_tasks}
  232. tasks2 = {k: v for k, v in tasks2.items() if k in only_tasks}
  233. if not tasks1:
  234. pkg_op = '+'
  235. elif not tasks2:
  236. pkg_op = '-'
  237. else:
  238. pkg_op = ' '
  239. for task in set(tasks1.keys()).union(set(tasks2.keys())):
  240. task_op = ' '
  241. if task in tasks1:
  242. val1 = getattr(bs1[pkg].tasks[task], stat_attr)
  243. else:
  244. task_op = '+'
  245. val1 = 0
  246. if task in tasks2:
  247. val2 = getattr(bs2[pkg].tasks[task], stat_attr)
  248. else:
  249. val2 = 0
  250. task_op = '-'
  251. if val1 == 0:
  252. reldiff = float('inf')
  253. else:
  254. reldiff = 100 * (val2 - val1) / val1
  255. if min_val and max(val1, val2) < min_val:
  256. log.debug("Filtering out %s:%s (%s)", pkg, task,
  257. max(val1, val2))
  258. continue
  259. if min_absdiff and abs(val2 - val1) < min_absdiff:
  260. log.debug("Filtering out %s:%s (difference of %s)", pkg, task,
  261. val2-val1)
  262. continue
  263. tasks_diff.append(TaskDiff(pkg, pkg_op, task, task_op, val1, val2,
  264. val2-val1, reldiff))
  265. return tasks_diff
  266. class BSVerDiff(object):
  267. """Class representing recipe version differences between two buildstats"""
  268. def __init__(self, bs1, bs2):
  269. RecipeVerDiff = namedtuple('RecipeVerDiff', 'left right')
  270. recipes1 = set(bs1.keys())
  271. recipes2 = set(bs2.keys())
  272. self.new = dict([(r, bs2[r]) for r in sorted(recipes2 - recipes1)])
  273. self.dropped = dict([(r, bs1[r]) for r in sorted(recipes1 - recipes2)])
  274. self.echanged = {}
  275. self.vchanged = {}
  276. self.rchanged = {}
  277. self.unchanged = {}
  278. self.empty_diff = False
  279. common = recipes2.intersection(recipes1)
  280. if common:
  281. for recipe in common:
  282. rdiff = RecipeVerDiff(bs1[recipe], bs2[recipe])
  283. if bs1[recipe].epoch != bs2[recipe].epoch:
  284. self.echanged[recipe] = rdiff
  285. elif bs1[recipe].version != bs2[recipe].version:
  286. self.vchanged[recipe] = rdiff
  287. elif bs1[recipe].revision != bs2[recipe].revision:
  288. self.rchanged[recipe] = rdiff
  289. else:
  290. self.unchanged[recipe] = rdiff
  291. if len(recipes1) == len(recipes2) == len(self.unchanged):
  292. self.empty_diff = True
  293. def __bool__(self):
  294. return not self.empty_diff