123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216 |
- #!/usr/bin/env python3
- #
- # Build performance test script
- #
- # Copyright (c) 2016, Intel Corporation.
- #
- # SPDX-License-Identifier: GPL-2.0-only
- #
- import argparse
- import errno
- import fcntl
- import json
- import logging
- import os
- import re
- import shutil
- import sys
- from datetime import datetime
- sys.path.insert(0, os.path.dirname(os.path.realpath(__file__)) + '/lib')
- import scriptpath
- scriptpath.add_oe_lib_path()
- scriptpath.add_bitbake_lib_path()
- import oeqa.buildperf
- from oeqa.buildperf import (BuildPerfTestLoader, BuildPerfTestResult,
- BuildPerfTestRunner, KernelDropCaches)
- from oeqa.utils.commands import runCmd
- from oeqa.utils.metadata import metadata_from_bb, write_metadata_file
- # Set-up logging
- LOG_FORMAT = '[%(asctime)s] %(levelname)s: %(message)s'
- logging.basicConfig(level=logging.INFO, format=LOG_FORMAT,
- datefmt='%Y-%m-%d %H:%M:%S')
- log = logging.getLogger()
- def acquire_lock(lock_f):
- """Acquire flock on file"""
- log.debug("Acquiring lock %s", os.path.abspath(lock_f.name))
- try:
- fcntl.flock(lock_f, fcntl.LOCK_EX | fcntl.LOCK_NB)
- except IOError as err:
- if err.errno == errno.EAGAIN:
- return False
- raise
- log.debug("Lock acquired")
- return True
- def pre_run_sanity_check():
- """Sanity check of build environment"""
- build_dir = os.environ.get("BUILDDIR")
- if not build_dir:
- log.error("BUILDDIR not set. Please run the build environmnent setup "
- "script.")
- return False
- if os.getcwd() != build_dir:
- log.error("Please run this script under BUILDDIR (%s)", build_dir)
- return False
- ret = runCmd('which bitbake', ignore_status=True)
- if ret.status:
- log.error("bitbake command not found")
- return False
- return True
- def setup_file_logging(log_file):
- """Setup loggin to file"""
- log_dir = os.path.dirname(log_file)
- if not os.path.exists(log_dir):
- os.makedirs(log_dir)
- formatter = logging.Formatter(LOG_FORMAT)
- handler = logging.FileHandler(log_file)
- handler.setFormatter(formatter)
- log.addHandler(handler)
- def archive_build_conf(out_dir):
- """Archive build/conf to test results"""
- src_dir = os.path.join(os.environ['BUILDDIR'], 'conf')
- tgt_dir = os.path.join(out_dir, 'build', 'conf')
- os.makedirs(os.path.dirname(tgt_dir))
- shutil.copytree(src_dir, tgt_dir)
- def update_globalres_file(result_obj, filename, metadata):
- """Write results to globalres csv file"""
- # Map test names to time and size columns in globalres
- # The tuples represent index and length of times and sizes
- # respectively
- gr_map = {'test1': ((0, 1), (8, 1)),
- 'test12': ((1, 1), (None, None)),
- 'test13': ((2, 1), (9, 1)),
- 'test2': ((3, 1), (None, None)),
- 'test3': ((4, 3), (None, None)),
- 'test4': ((7, 1), (10, 2))}
- values = ['0'] * 12
- for status, test, _ in result_obj.all_results():
- if status in ['ERROR', 'SKIPPED']:
- continue
- (t_ind, t_len), (s_ind, s_len) = gr_map[test.name]
- if t_ind is not None:
- values[t_ind:t_ind + t_len] = test.times
- if s_ind is not None:
- values[s_ind:s_ind + s_len] = test.sizes
- log.debug("Writing globalres log to %s", filename)
- rev_info = metadata['layers']['meta']
- with open(filename, 'a') as fobj:
- fobj.write('{},{}:{},{},'.format(metadata['hostname'],
- rev_info['branch'],
- rev_info['commit'],
- rev_info['commit']))
- fobj.write(','.join(values) + '\n')
- def parse_args(argv):
- """Parse command line arguments"""
- parser = argparse.ArgumentParser(
- formatter_class=argparse.ArgumentDefaultsHelpFormatter)
- parser.add_argument('-D', '--debug', action='store_true',
- help='Enable debug level logging')
- parser.add_argument('--globalres-file',
- type=os.path.abspath,
- help="Append results to 'globalres' csv file")
- parser.add_argument('--lock-file', default='./oe-build-perf.lock',
- metavar='FILENAME', type=os.path.abspath,
- help="Lock file to use")
- parser.add_argument('-o', '--out-dir', default='results-{date}',
- type=os.path.abspath,
- help="Output directory for test results")
- parser.add_argument('-x', '--xml', action='store_true',
- help='Enable JUnit xml output')
- parser.add_argument('--log-file',
- default='{out_dir}/oe-build-perf-test.log',
- help="Log file of this script")
- parser.add_argument('--run-tests', nargs='+', metavar='TEST',
- help="List of tests to run")
- return parser.parse_args(argv)
- def main(argv=None):
- """Script entry point"""
- args = parse_args(argv)
- # Set-up log file
- out_dir = args.out_dir.format(date=datetime.now().strftime('%Y%m%d%H%M%S'))
- setup_file_logging(args.log_file.format(out_dir=out_dir))
- if args.debug:
- log.setLevel(logging.DEBUG)
- lock_f = open(args.lock_file, 'w')
- if not acquire_lock(lock_f):
- log.error("Another instance of this script is running, exiting...")
- return 1
- if not pre_run_sanity_check():
- return 1
- # Check our capability to drop caches and ask pass if needed
- KernelDropCaches.check()
- # Load build perf tests
- loader = BuildPerfTestLoader()
- if args.run_tests:
- suite = loader.loadTestsFromNames(args.run_tests, oeqa.buildperf)
- else:
- suite = loader.loadTestsFromModule(oeqa.buildperf)
- # Save test metadata
- metadata = metadata_from_bb()
- log.info("Testing Git revision branch:commit %s:%s (%s)",
- metadata['layers']['meta']['branch'],
- metadata['layers']['meta']['commit'],
- metadata['layers']['meta']['commit_count'])
- if args.xml:
- write_metadata_file(os.path.join(out_dir, 'metadata.xml'), metadata)
- else:
- with open(os.path.join(out_dir, 'metadata.json'), 'w') as fobj:
- json.dump(metadata, fobj, indent=2)
- archive_build_conf(out_dir)
- runner = BuildPerfTestRunner(out_dir, verbosity=2)
- # Suppress logger output to stderr so that the output from unittest
- # is not mixed with occasional logger output
- log.handlers[0].setLevel(logging.CRITICAL)
- # Run actual tests
- result = runner.run(suite)
- # Restore logger output to stderr
- log.handlers[0].setLevel(log.level)
- if args.xml:
- result.write_results_xml()
- else:
- result.write_results_json()
- result.write_buildstats_json()
- if args.globalres_file:
- update_globalres_file(result, args.globalres_file, metadata)
- if result.wasSuccessful():
- return 0
- return 2
- if __name__ == '__main__':
- sys.exit(main())
|