123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615 |
- #!/usr/bin/env python3
- #
- # Examine build performance test results
- #
- # Copyright (c) 2017, Intel Corporation.
- #
- # SPDX-License-Identifier: GPL-2.0-only
- #
- import argparse
- import json
- import logging
- import os
- import re
- import sys
- from collections import namedtuple, OrderedDict
- from operator import attrgetter
- from xml.etree import ElementTree as ET
- # Import oe libs
- scripts_path = os.path.dirname(os.path.realpath(__file__))
- sys.path.append(os.path.join(scripts_path, 'lib'))
- import scriptpath
- from build_perf import print_table
- from build_perf.report import (metadata_xml_to_json, results_xml_to_json,
- aggregate_data, aggregate_metadata, measurement_stats,
- AggregateTestData)
- from build_perf import html
- from buildstats import BuildStats, diff_buildstats, BSVerDiff
- scriptpath.add_oe_lib_path()
- from oeqa.utils.git import GitRepo, GitError
- import oeqa.utils.gitarchive as gitarchive
- # Setup logging
- logging.basicConfig(level=logging.INFO, format="%(levelname)s: %(message)s")
- log = logging.getLogger('oe-build-perf-report')
- def list_test_revs(repo, tag_name, verbosity, **kwargs):
- """Get list of all tested revisions"""
- valid_kwargs = dict([(k, v) for k, v in kwargs.items() if v is not None])
- fields, revs = gitarchive.get_test_runs(log, repo, tag_name, **valid_kwargs)
- ignore_fields = ['tag_number']
- if verbosity < 2:
- extra_fields = ['COMMITS', 'TEST RUNS']
- ignore_fields.extend(['commit_number', 'commit'])
- else:
- extra_fields = ['TEST RUNS']
- print_fields = [i for i, f in enumerate(fields) if f not in ignore_fields]
- # Sort revs
- rows = [[fields[i].upper() for i in print_fields] + extra_fields]
- prev = [''] * len(print_fields)
- prev_commit = None
- commit_cnt = 0
- commit_field = fields.index('commit')
- for rev in revs:
- # Only use fields that we want to print
- cols = [rev[i] for i in print_fields]
- if cols != prev:
- commit_cnt = 1
- test_run_cnt = 1
- new_row = [''] * (len(print_fields) + len(extra_fields))
- for i in print_fields:
- if cols[i] != prev[i]:
- break
- new_row[i:-len(extra_fields)] = cols[i:]
- rows.append(new_row)
- else:
- if rev[commit_field] != prev_commit:
- commit_cnt += 1
- test_run_cnt += 1
- if verbosity < 2:
- new_row[-2] = commit_cnt
- new_row[-1] = test_run_cnt
- prev = cols
- prev_commit = rev[commit_field]
- print_table(rows)
- def is_xml_format(repo, commit):
- """Check if the commit contains xml (or json) data"""
- if repo.rev_parse(commit + ':results.xml'):
- log.debug("Detected report in xml format in %s", commit)
- return True
- else:
- log.debug("No xml report in %s, assuming json formatted results", commit)
- return False
- def read_results(repo, tags, xml=True):
- """Read result files from repo"""
- def parse_xml_stream(data):
- """Parse multiple concatenated XML objects"""
- objs = []
- xml_d = ""
- for line in data.splitlines():
- if xml_d and line.startswith('<?xml version='):
- objs.append(ET.fromstring(xml_d))
- xml_d = line
- else:
- xml_d += line
- objs.append(ET.fromstring(xml_d))
- return objs
- def parse_json_stream(data):
- """Parse multiple concatenated JSON objects"""
- objs = []
- json_d = ""
- for line in data.splitlines():
- if line == '}{':
- json_d += '}'
- objs.append(json.loads(json_d, object_pairs_hook=OrderedDict))
- json_d = '{'
- else:
- json_d += line
- objs.append(json.loads(json_d, object_pairs_hook=OrderedDict))
- return objs
- num_revs = len(tags)
- # Optimize by reading all data with one git command
- log.debug("Loading raw result data from %d tags, %s...", num_revs, tags[0])
- if xml:
- git_objs = [tag + ':metadata.xml' for tag in tags] + [tag + ':results.xml' for tag in tags]
- data = parse_xml_stream(repo.run_cmd(['show'] + git_objs + ['--']))
- return ([metadata_xml_to_json(e) for e in data[0:num_revs]],
- [results_xml_to_json(e) for e in data[num_revs:]])
- else:
- git_objs = [tag + ':metadata.json' for tag in tags] + [tag + ':results.json' for tag in tags]
- data = parse_json_stream(repo.run_cmd(['show'] + git_objs + ['--']))
- return data[0:num_revs], data[num_revs:]
- def get_data_item(data, key):
- """Nested getitem lookup"""
- for k in key.split('.'):
- data = data[k]
- return data
- def metadata_diff(metadata_l, metadata_r):
- """Prepare a metadata diff for printing"""
- keys = [('Hostname', 'hostname', 'hostname'),
- ('Branch', 'branch', 'layers.meta.branch'),
- ('Commit number', 'commit_num', 'layers.meta.commit_count'),
- ('Commit', 'commit', 'layers.meta.commit'),
- ('Number of test runs', 'testrun_count', 'testrun_count')
- ]
- def _metadata_diff(key):
- """Diff metadata from two test reports"""
- try:
- val1 = get_data_item(metadata_l, key)
- except KeyError:
- val1 = '(N/A)'
- try:
- val2 = get_data_item(metadata_r, key)
- except KeyError:
- val2 = '(N/A)'
- return val1, val2
- metadata = OrderedDict()
- for title, key, key_json in keys:
- value_l, value_r = _metadata_diff(key_json)
- metadata[key] = {'title': title,
- 'value_old': value_l,
- 'value': value_r}
- return metadata
- def print_diff_report(metadata_l, data_l, metadata_r, data_r):
- """Print differences between two data sets"""
- # First, print general metadata
- print("\nTEST METADATA:\n==============")
- meta_diff = metadata_diff(metadata_l, metadata_r)
- rows = []
- row_fmt = ['{:{wid}} ', '{:<{wid}} ', '{:<{wid}}']
- rows = [['', 'CURRENT COMMIT', 'COMPARING WITH']]
- for key, val in meta_diff.items():
- # Shorten commit hashes
- if key == 'commit':
- rows.append([val['title'] + ':', val['value'][:20], val['value_old'][:20]])
- else:
- rows.append([val['title'] + ':', val['value'], val['value_old']])
- print_table(rows, row_fmt)
- # Print test results
- print("\nTEST RESULTS:\n=============")
- tests = list(data_l['tests'].keys())
- # Append tests that are only present in 'right' set
- tests += [t for t in list(data_r['tests'].keys()) if t not in tests]
- # Prepare data to be printed
- rows = []
- row_fmt = ['{:8}', '{:{wid}}', '{:{wid}}', ' {:>{wid}}', ' {:{wid}} ', '{:{wid}}',
- ' {:>{wid}}', ' {:>{wid}}']
- num_cols = len(row_fmt)
- for test in tests:
- test_l = data_l['tests'][test] if test in data_l['tests'] else None
- test_r = data_r['tests'][test] if test in data_r['tests'] else None
- pref = ' '
- if test_l is None:
- pref = '+'
- elif test_r is None:
- pref = '-'
- descr = test_l['description'] if test_l else test_r['description']
- heading = "{} {}: {}".format(pref, test, descr)
- rows.append([heading])
- # Generate the list of measurements
- meas_l = test_l['measurements'] if test_l else {}
- meas_r = test_r['measurements'] if test_r else {}
- measurements = list(meas_l.keys())
- measurements += [m for m in list(meas_r.keys()) if m not in measurements]
- for meas in measurements:
- m_pref = ' '
- if meas in meas_l:
- stats_l = measurement_stats(meas_l[meas], 'l.')
- else:
- stats_l = measurement_stats(None, 'l.')
- m_pref = '+'
- if meas in meas_r:
- stats_r = measurement_stats(meas_r[meas], 'r.')
- else:
- stats_r = measurement_stats(None, 'r.')
- m_pref = '-'
- stats = stats_l.copy()
- stats.update(stats_r)
- absdiff = stats['val_cls'](stats['r.mean'] - stats['l.mean'])
- reldiff = "{:+.1f} %".format(absdiff * 100 / stats['l.mean'])
- if stats['r.mean'] > stats['l.mean']:
- absdiff = '+' + str(absdiff)
- else:
- absdiff = str(absdiff)
- rows.append(['', m_pref, stats['name'] + ' ' + stats['quantity'],
- str(stats['l.mean']), '->', str(stats['r.mean']),
- absdiff, reldiff])
- rows.append([''] * num_cols)
- print_table(rows, row_fmt)
- print()
- class BSSummary(object):
- def __init__(self, bs1, bs2):
- self.tasks = {'count': bs2.num_tasks,
- 'change': '{:+d}'.format(bs2.num_tasks - bs1.num_tasks)}
- self.top_consumer = None
- self.top_decrease = None
- self.top_increase = None
- self.ver_diff = OrderedDict()
- tasks_diff = diff_buildstats(bs1, bs2, 'cputime')
- # Get top consumers of resources
- tasks_diff = sorted(tasks_diff, key=attrgetter('value2'))
- self.top_consumer = tasks_diff[-5:]
- # Get biggest increase and decrease in resource usage
- tasks_diff = sorted(tasks_diff, key=attrgetter('absdiff'))
- self.top_decrease = tasks_diff[0:5]
- self.top_increase = tasks_diff[-5:]
- # Compare recipe versions and prepare data for display
- ver_diff = BSVerDiff(bs1, bs2)
- if ver_diff:
- if ver_diff.new:
- self.ver_diff['New recipes'] = [(n, r.evr) for n, r in ver_diff.new.items()]
- if ver_diff.dropped:
- self.ver_diff['Dropped recipes'] = [(n, r.evr) for n, r in ver_diff.dropped.items()]
- if ver_diff.echanged:
- self.ver_diff['Epoch changed'] = [(n, "{} → {}".format(r.left.evr, r.right.evr)) for n, r in ver_diff.echanged.items()]
- if ver_diff.vchanged:
- self.ver_diff['Version changed'] = [(n, "{} → {}".format(r.left.version, r.right.version)) for n, r in ver_diff.vchanged.items()]
- if ver_diff.rchanged:
- self.ver_diff['Revision changed'] = [(n, "{} → {}".format(r.left.evr, r.right.evr)) for n, r in ver_diff.rchanged.items()]
- def print_html_report(data, id_comp, buildstats):
- """Print report in html format"""
- # Handle metadata
- metadata = metadata_diff(data[id_comp].metadata, data[-1].metadata)
- # Generate list of tests
- tests = []
- for test in data[-1].results['tests'].keys():
- test_r = data[-1].results['tests'][test]
- new_test = {'name': test_r['name'],
- 'description': test_r['description'],
- 'status': test_r['status'],
- 'measurements': [],
- 'err_type': test_r.get('err_type'),
- }
- # Limit length of err output shown
- if 'message' in test_r:
- lines = test_r['message'].splitlines()
- if len(lines) > 20:
- new_test['message'] = '...\n' + '\n'.join(lines[-20:])
- else:
- new_test['message'] = test_r['message']
- # Generate the list of measurements
- for meas in test_r['measurements'].keys():
- meas_r = test_r['measurements'][meas]
- meas_type = 'time' if meas_r['type'] == 'sysres' else 'size'
- new_meas = {'name': meas_r['name'],
- 'legend': meas_r['legend'],
- 'description': meas_r['name'] + ' ' + meas_type,
- }
- samples = []
- # Run through all revisions in our data
- for meta, test_data in data:
- if (not test in test_data['tests'] or
- not meas in test_data['tests'][test]['measurements']):
- samples.append(measurement_stats(None))
- continue
- test_i = test_data['tests'][test]
- meas_i = test_i['measurements'][meas]
- commit_num = get_data_item(meta, 'layers.meta.commit_count')
- samples.append(measurement_stats(meas_i))
- samples[-1]['commit_num'] = commit_num
- absdiff = samples[-1]['val_cls'](samples[-1]['mean'] - samples[id_comp]['mean'])
- reldiff = absdiff * 100 / samples[id_comp]['mean']
- new_meas['absdiff'] = absdiff
- new_meas['absdiff_str'] = str(absdiff) if absdiff < 0 else '+' + str(absdiff)
- new_meas['reldiff'] = reldiff
- new_meas['reldiff_str'] = "{:+.1f} %".format(reldiff)
- new_meas['samples'] = samples
- new_meas['value'] = samples[-1]
- new_meas['value_type'] = samples[-1]['val_cls']
- # Compare buildstats
- bs_key = test + '.' + meas
- rev = str(metadata['commit_num']['value'])
- comp_rev = str(metadata['commit_num']['value_old'])
- if (buildstats and rev in buildstats and bs_key in buildstats[rev] and
- comp_rev in buildstats and bs_key in buildstats[comp_rev]):
- new_meas['buildstats'] = BSSummary(buildstats[comp_rev][bs_key],
- buildstats[rev][bs_key])
- new_test['measurements'].append(new_meas)
- tests.append(new_test)
- # Chart options
- chart_opts = {'haxis': {'min': get_data_item(data[0][0], 'layers.meta.commit_count'),
- 'max': get_data_item(data[-1][0], 'layers.meta.commit_count')}
- }
- print(html.template.render(title="Build Perf Test Report",
- metadata=metadata, test_data=tests,
- chart_opts=chart_opts))
- def get_buildstats(repo, notes_ref, notes_ref2, revs, outdir=None):
- """Get the buildstats from git notes"""
- full_ref = 'refs/notes/' + notes_ref
- if not repo.rev_parse(full_ref):
- log.error("No buildstats found, please try running "
- "'git fetch origin %s:%s' to fetch them from the remote",
- full_ref, full_ref)
- return
- missing = False
- buildstats = {}
- log.info("Parsing buildstats from 'refs/notes/%s'", notes_ref)
- for rev in revs:
- buildstats[rev.commit_number] = {}
- log.debug('Dumping buildstats for %s (%s)', rev.commit_number,
- rev.commit)
- for tag in rev.tags:
- log.debug(' %s', tag)
- try:
- try:
- bs_all = json.loads(repo.run_cmd(['notes', '--ref', notes_ref, 'show', tag + '^0']))
- except GitError:
- if notes_ref2:
- bs_all = json.loads(repo.run_cmd(['notes', '--ref', notes_ref2, 'show', tag + '^0']))
- else:
- raise
- except GitError:
- log.warning("Buildstats not found for %s", tag)
- bs_all = {}
- missing = True
- for measurement, bs in bs_all.items():
- # Write out onto disk
- if outdir:
- tag_base, run_id = tag.rsplit('/', 1)
- tag_base = tag_base.replace('/', '_')
- bs_dir = os.path.join(outdir, measurement, tag_base)
- if not os.path.exists(bs_dir):
- os.makedirs(bs_dir)
- with open(os.path.join(bs_dir, run_id + '.json'), 'w') as f:
- json.dump(bs, f, indent=2)
- # Read buildstats into a dict
- _bs = BuildStats.from_json(bs)
- if measurement not in buildstats[rev.commit_number]:
- buildstats[rev.commit_number][measurement] = _bs
- else:
- buildstats[rev.commit_number][measurement].aggregate(_bs)
- if missing:
- log.info("Buildstats were missing for some test runs, please "
- "run 'git fetch origin %s:%s' and try again",
- full_ref, full_ref)
- return buildstats
- def auto_args(repo, args):
- """Guess arguments, if not defined by the user"""
- # Get the latest commit in the repo
- log.debug("Guessing arguments from the latest commit")
- msg = repo.run_cmd(['log', '-1', '--branches', '--remotes', '--format=%b'])
- for line in msg.splitlines():
- split = line.split(':', 1)
- if len(split) != 2:
- continue
- key = split[0]
- val = split[1].strip()
- if key == 'hostname' and not args.hostname:
- log.debug("Using hostname %s", val)
- args.hostname = val
- elif key == 'branch' and not args.branch:
- log.debug("Using branch %s", val)
- args.branch = val
- def parse_args(argv):
- """Parse command line arguments"""
- description = """
- Examine build performance test results from a Git repository"""
- parser = argparse.ArgumentParser(
- formatter_class=argparse.ArgumentDefaultsHelpFormatter,
- description=description)
- parser.add_argument('--debug', '-d', action='store_true',
- help="Verbose logging")
- parser.add_argument('--repo', '-r', required=True,
- help="Results repository (local git clone)")
- parser.add_argument('--list', '-l', action='count',
- help="List available test runs")
- parser.add_argument('--html', action='store_true',
- help="Generate report in html format")
- group = parser.add_argument_group('Tag and revision')
- group.add_argument('--tag-name', '-t',
- default='{hostname}/{branch}/{machine}/{commit_number}-g{commit}/{tag_number}',
- help="Tag name (pattern) for finding results")
- group.add_argument('--hostname', '-H')
- group.add_argument('--branch', '-B', default='master', help="Branch to find commit in")
- group.add_argument('--branch2', help="Branch to find comparision revisions in")
- group.add_argument('--machine', default='qemux86')
- group.add_argument('--history-length', default=25, type=int,
- help="Number of tested revisions to plot in html report")
- group.add_argument('--commit',
- help="Revision to search for")
- group.add_argument('--commit-number',
- help="Revision number to search for, redundant if "
- "--commit is specified")
- group.add_argument('--commit2',
- help="Revision to compare with")
- group.add_argument('--commit-number2',
- help="Revision number to compare with, redundant if "
- "--commit2 is specified")
- parser.add_argument('--dump-buildstats', nargs='?', const='.',
- help="Dump buildstats of the tests")
- return parser.parse_args(argv)
- def main(argv=None):
- """Script entry point"""
- args = parse_args(argv)
- if args.debug:
- log.setLevel(logging.DEBUG)
- repo = GitRepo(args.repo)
- if args.list:
- list_test_revs(repo, args.tag_name, args.list, hostname=args.hostname)
- return 0
- # Determine hostname which to use
- if not args.hostname:
- auto_args(repo, args)
- revs = gitarchive.get_test_revs(log, repo, args.tag_name, hostname=args.hostname,
- branch=args.branch, machine=args.machine)
- if args.branch2:
- revs2 = gitarchive.get_test_revs(log, repo, args.tag_name, hostname=args.hostname,
- branch=args.branch2, machine=args.machine)
- if not len(revs2):
- log.error("No revisions found to compare against")
- return 1
- if not len(revs):
- log.error("No revision to report on found")
- return 1
- else:
- if len(revs) < 2:
- log.error("Only %d tester revisions found, unable to generate report" % len(revs))
- return 1
- # Pick revisions
- if args.commit:
- if args.commit_number:
- log.warning("Ignoring --commit-number as --commit was specified")
- index1 = gitarchive.rev_find(revs, 'commit', args.commit)
- elif args.commit_number:
- index1 = gitarchive.rev_find(revs, 'commit_number', args.commit_number)
- else:
- index1 = len(revs) - 1
- if args.branch2:
- revs2.append(revs[index1])
- index1 = len(revs2) - 1
- revs = revs2
- if args.commit2:
- if args.commit_number2:
- log.warning("Ignoring --commit-number2 as --commit2 was specified")
- index2 = gitarchive.rev_find(revs, 'commit', args.commit2)
- elif args.commit_number2:
- index2 = gitarchive.rev_find(revs, 'commit_number', args.commit_number2)
- else:
- if index1 > 0:
- index2 = index1 - 1
- # Find the closest matching commit number for comparision
- # In future we could check the commit is a common ancestor and
- # continue back if not but this good enough for now
- while index2 > 0 and revs[index2].commit_number > revs[index1].commit_number:
- index2 = index2 - 1
- else:
- log.error("Unable to determine the other commit, use "
- "--commit2 or --commit-number2 to specify it")
- return 1
- index_l = min(index1, index2)
- index_r = max(index1, index2)
- rev_l = revs[index_l]
- rev_r = revs[index_r]
- log.debug("Using 'left' revision %s (%s), %s test runs:\n %s",
- rev_l.commit_number, rev_l.commit, len(rev_l.tags),
- '\n '.join(rev_l.tags))
- log.debug("Using 'right' revision %s (%s), %s test runs:\n %s",
- rev_r.commit_number, rev_r.commit, len(rev_r.tags),
- '\n '.join(rev_r.tags))
- # Check report format used in the repo (assume all reports in the same fmt)
- xml = is_xml_format(repo, revs[index_r].tags[-1])
- if args.html:
- index_0 = max(0, min(index_l, index_r - args.history_length))
- rev_range = range(index_0, index_r + 1)
- else:
- # We do not need range of commits for text report (no graphs)
- index_0 = index_l
- rev_range = (index_l, index_r)
- # Read raw data
- log.debug("Reading %d revisions, starting from %s (%s)",
- len(rev_range), revs[index_0].commit_number, revs[index_0].commit)
- raw_data = [read_results(repo, revs[i].tags, xml) for i in rev_range]
- data = []
- for raw_m, raw_d in raw_data:
- data.append(AggregateTestData(aggregate_metadata(raw_m),
- aggregate_data(raw_d)))
- # Read buildstats only when needed
- buildstats = None
- if args.dump_buildstats or args.html:
- outdir = 'oe-build-perf-buildstats' if args.dump_buildstats else None
- notes_ref = 'buildstats/{}/{}/{}'.format(args.hostname, args.branch, args.machine)
- notes_ref2 = None
- if args.branch2:
- notes_ref = 'buildstats/{}/{}/{}'.format(args.hostname, args.branch2, args.machine)
- notes_ref2 = 'buildstats/{}/{}/{}'.format(args.hostname, args.branch, args.machine)
- buildstats = get_buildstats(repo, notes_ref, notes_ref2, [rev_l, rev_r], outdir)
- # Print report
- if not args.html:
- print_diff_report(data[0].metadata, data[0].results,
- data[1].metadata, data[1].results)
- else:
- # Re-map 'left' list index to the data table where index_0 maps to 0
- print_html_report(data, index_l - index_0, buildstats)
- return 0
- if __name__ == "__main__":
- sys.exit(main())
|