123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346 |
- #
- # Copyright (c) 2017, Intel Corporation.
- #
- # SPDX-License-Identifier: GPL-2.0-only
- #
- """Functionality for analyzing buildstats"""
- import json
- import logging
- import os
- import re
- from collections import namedtuple,OrderedDict
- from statistics import mean
- log = logging.getLogger()
- taskdiff_fields = ('pkg', 'pkg_op', 'task', 'task_op', 'value1', 'value2',
- 'absdiff', 'reldiff')
- TaskDiff = namedtuple('TaskDiff', ' '.join(taskdiff_fields))
- class BSError(Exception):
- """Error handling of buildstats"""
- pass
- class BSTask(dict):
- def __init__(self, *args, **kwargs):
- self['start_time'] = None
- self['elapsed_time'] = None
- self['status'] = None
- self['iostat'] = {}
- self['rusage'] = {}
- self['child_rusage'] = {}
- super(BSTask, self).__init__(*args, **kwargs)
- @property
- def cputime(self):
- """Sum of user and system time taken by the task"""
- rusage = self['rusage']['ru_stime'] + self['rusage']['ru_utime']
- if self['child_rusage']:
- # Child rusage may have been optimized out
- return rusage + self['child_rusage']['ru_stime'] + self['child_rusage']['ru_utime']
- else:
- return rusage
- @property
- def walltime(self):
- """Elapsed wall clock time"""
- return self['elapsed_time']
- @property
- def read_bytes(self):
- """Bytes read from the block layer"""
- return self['iostat']['read_bytes']
- @property
- def write_bytes(self):
- """Bytes written to the block layer"""
- return self['iostat']['write_bytes']
- @property
- def read_ops(self):
- """Number of read operations on the block layer"""
- if self['child_rusage']:
- # Child rusage may have been optimized out
- return self['rusage']['ru_inblock'] + self['child_rusage']['ru_inblock']
- else:
- return self['rusage']['ru_inblock']
- @property
- def write_ops(self):
- """Number of write operations on the block layer"""
- if self['child_rusage']:
- # Child rusage may have been optimized out
- return self['rusage']['ru_oublock'] + self['child_rusage']['ru_oublock']
- else:
- return self['rusage']['ru_oublock']
- @classmethod
- def from_file(cls, buildstat_file):
- """Read buildstat text file"""
- bs_task = cls()
- log.debug("Reading task buildstats from %s", buildstat_file)
- end_time = None
- with open(buildstat_file) as fobj:
- for line in fobj.readlines():
- key, val = line.split(':', 1)
- val = val.strip()
- if key == 'Started':
- start_time = float(val)
- bs_task['start_time'] = start_time
- elif key == 'Ended':
- end_time = float(val)
- elif key.startswith('IO '):
- split = key.split()
- bs_task['iostat'][split[1]] = int(val)
- elif key.find('rusage') >= 0:
- split = key.split()
- ru_key = split[-1]
- if ru_key in ('ru_stime', 'ru_utime'):
- val = float(val)
- else:
- val = int(val)
- ru_type = 'rusage' if split[0] == 'rusage' else \
- 'child_rusage'
- bs_task[ru_type][ru_key] = val
- elif key == 'Status':
- bs_task['status'] = val
- if end_time is not None and start_time is not None:
- bs_task['elapsed_time'] = end_time - start_time
- else:
- raise BSError("{} looks like a invalid buildstats file".format(buildstat_file))
- return bs_task
- class BSTaskAggregate(object):
- """Class representing multiple runs of the same task"""
- properties = ('cputime', 'walltime', 'read_bytes', 'write_bytes',
- 'read_ops', 'write_ops')
- def __init__(self, tasks=None):
- self._tasks = tasks or []
- self._properties = {}
- def __getattr__(self, name):
- if name in self.properties:
- if name not in self._properties:
- # Calculate properties on demand only. We only provide mean
- # value, so far
- self._properties[name] = mean([getattr(t, name) for t in self._tasks])
- return self._properties[name]
- else:
- raise AttributeError("'BSTaskAggregate' has no attribute '{}'".format(name))
- def append(self, task):
- """Append new task"""
- # Reset pre-calculated properties
- assert isinstance(task, BSTask), "Type is '{}' instead of 'BSTask'".format(type(task))
- self._properties = {}
- self._tasks.append(task)
- class BSRecipe(object):
- """Class representing buildstats of one recipe"""
- def __init__(self, name, epoch, version, revision):
- self.name = name
- self.epoch = epoch
- self.version = version
- self.revision = revision
- if epoch is None:
- self.evr = "{}-{}".format(version, revision)
- else:
- self.evr = "{}_{}-{}".format(epoch, version, revision)
- self.tasks = {}
- def aggregate(self, bsrecipe):
- """Aggregate data of another recipe buildstats"""
- if self.nevr != bsrecipe.nevr:
- raise ValueError("Refusing to aggregate buildstats, recipe version "
- "differs: {} vs. {}".format(self.nevr, bsrecipe.nevr))
- if set(self.tasks.keys()) != set(bsrecipe.tasks.keys()):
- raise ValueError("Refusing to aggregate buildstats, set of tasks "
- "in {} differ".format(self.name))
- for taskname, taskdata in bsrecipe.tasks.items():
- if not isinstance(self.tasks[taskname], BSTaskAggregate):
- self.tasks[taskname] = BSTaskAggregate([self.tasks[taskname]])
- self.tasks[taskname].append(taskdata)
- @property
- def nevr(self):
- return self.name + '-' + self.evr
- class BuildStats(dict):
- """Class representing buildstats of one build"""
- @property
- def num_tasks(self):
- """Get number of tasks"""
- num = 0
- for recipe in self.values():
- num += len(recipe.tasks)
- return num
- @classmethod
- def from_json(cls, bs_json):
- """Create new BuildStats object from JSON object"""
- buildstats = cls()
- for recipe in bs_json:
- if recipe['name'] in buildstats:
- raise BSError("Cannot handle multiple versions of the same "
- "package ({})".format(recipe['name']))
- bsrecipe = BSRecipe(recipe['name'], recipe['epoch'],
- recipe['version'], recipe['revision'])
- for task, data in recipe['tasks'].items():
- bsrecipe.tasks[task] = BSTask(data)
- buildstats[recipe['name']] = bsrecipe
- return buildstats
- @staticmethod
- def from_file_json(path):
- """Load buildstats from a JSON file"""
- with open(path) as fobj:
- bs_json = json.load(fobj)
- return BuildStats.from_json(bs_json)
- @staticmethod
- def split_nevr(nevr):
- """Split name and version information from recipe "nevr" string"""
- n_e_v, revision = nevr.rsplit('-', 1)
- match = re.match(r'^(?P<name>\S+)-((?P<epoch>[0-9]{1,5})_)?(?P<version>[0-9]\S*)$',
- n_e_v)
- if not match:
- # If we're not able to parse a version starting with a number, just
- # take the part after last dash
- match = re.match(r'^(?P<name>\S+)-((?P<epoch>[0-9]{1,5})_)?(?P<version>[^-]+)$',
- n_e_v)
- name = match.group('name')
- version = match.group('version')
- epoch = match.group('epoch')
- return name, epoch, version, revision
- @classmethod
- def from_dir(cls, path):
- """Load buildstats from a buildstats directory"""
- if not os.path.isfile(os.path.join(path, 'build_stats')):
- raise BSError("{} does not look like a buildstats directory".format(path))
- log.debug("Reading buildstats directory %s", path)
- buildstats = cls()
- subdirs = os.listdir(path)
- for dirname in subdirs:
- recipe_dir = os.path.join(path, dirname)
- if not os.path.isdir(recipe_dir):
- continue
- name, epoch, version, revision = cls.split_nevr(dirname)
- bsrecipe = BSRecipe(name, epoch, version, revision)
- for task in os.listdir(recipe_dir):
- bsrecipe.tasks[task] = BSTask.from_file(
- os.path.join(recipe_dir, task))
- if name in buildstats:
- raise BSError("Cannot handle multiple versions of the same "
- "package ({})".format(name))
- buildstats[name] = bsrecipe
- return buildstats
- def aggregate(self, buildstats):
- """Aggregate other buildstats into this"""
- if set(self.keys()) != set(buildstats.keys()):
- raise ValueError("Refusing to aggregate buildstats, set of "
- "recipes is different: %s" % (set(self.keys()) ^ set(buildstats.keys())))
- for pkg, data in buildstats.items():
- self[pkg].aggregate(data)
- def diff_buildstats(bs1, bs2, stat_attr, min_val=None, min_absdiff=None, only_tasks=[]):
- """Compare the tasks of two buildstats"""
- tasks_diff = []
- pkgs = set(bs1.keys()).union(set(bs2.keys()))
- for pkg in pkgs:
- tasks1 = bs1[pkg].tasks if pkg in bs1 else {}
- tasks2 = bs2[pkg].tasks if pkg in bs2 else {}
- if only_tasks:
- tasks1 = {k: v for k, v in tasks1.items() if k in only_tasks}
- tasks2 = {k: v for k, v in tasks2.items() if k in only_tasks}
- if not tasks1:
- pkg_op = '+'
- elif not tasks2:
- pkg_op = '-'
- else:
- pkg_op = ' '
- for task in set(tasks1.keys()).union(set(tasks2.keys())):
- task_op = ' '
- if task in tasks1:
- val1 = getattr(bs1[pkg].tasks[task], stat_attr)
- else:
- task_op = '+'
- val1 = 0
- if task in tasks2:
- val2 = getattr(bs2[pkg].tasks[task], stat_attr)
- else:
- val2 = 0
- task_op = '-'
- if val1 == 0:
- reldiff = float('inf')
- else:
- reldiff = 100 * (val2 - val1) / val1
- if min_val and max(val1, val2) < min_val:
- log.debug("Filtering out %s:%s (%s)", pkg, task,
- max(val1, val2))
- continue
- if min_absdiff and abs(val2 - val1) < min_absdiff:
- log.debug("Filtering out %s:%s (difference of %s)", pkg, task,
- val2-val1)
- continue
- tasks_diff.append(TaskDiff(pkg, pkg_op, task, task_op, val1, val2,
- val2-val1, reldiff))
- return tasks_diff
- class BSVerDiff(object):
- """Class representing recipe version differences between two buildstats"""
- def __init__(self, bs1, bs2):
- RecipeVerDiff = namedtuple('RecipeVerDiff', 'left right')
- recipes1 = set(bs1.keys())
- recipes2 = set(bs2.keys())
- self.new = dict([(r, bs2[r]) for r in sorted(recipes2 - recipes1)])
- self.dropped = dict([(r, bs1[r]) for r in sorted(recipes1 - recipes2)])
- self.echanged = {}
- self.vchanged = {}
- self.rchanged = {}
- self.unchanged = {}
- self.empty_diff = False
- common = recipes2.intersection(recipes1)
- if common:
- for recipe in common:
- rdiff = RecipeVerDiff(bs1[recipe], bs2[recipe])
- if bs1[recipe].epoch != bs2[recipe].epoch:
- self.echanged[recipe] = rdiff
- elif bs1[recipe].version != bs2[recipe].version:
- self.vchanged[recipe] = rdiff
- elif bs1[recipe].revision != bs2[recipe].revision:
- self.rchanged[recipe] = rdiff
- else:
- self.unchanged[recipe] = rdiff
- if len(recipes1) == len(recipes2) == len(self.unchanged):
- self.empty_diff = True
- def __bool__(self):
- return not self.empty_diff
|