oe-build-perf-test 7.3 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223
  1. #!/usr/bin/python3
  2. #
  3. # Build performance test script
  4. #
  5. # Copyright (c) 2016, Intel Corporation.
  6. #
  7. # This program is free software; you can redistribute it and/or modify it
  8. # under the terms and conditions of the GNU General Public License,
  9. # version 2, as published by the Free Software Foundation.
  10. #
  11. # This program is distributed in the hope it will be useful, but WITHOUT
  12. # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  13. # FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
  14. # more details.
  15. #
  16. """Build performance test script"""
  17. import argparse
  18. import errno
  19. import fcntl
  20. import json
  21. import logging
  22. import os
  23. import re
  24. import shutil
  25. import sys
  26. from datetime import datetime
  27. sys.path.insert(0, os.path.dirname(os.path.realpath(__file__)) + '/lib')
  28. import scriptpath
  29. scriptpath.add_oe_lib_path()
  30. scriptpath.add_bitbake_lib_path()
  31. import oeqa.buildperf
  32. from oeqa.buildperf import (BuildPerfTestLoader, BuildPerfTestResult,
  33. BuildPerfTestRunner, KernelDropCaches)
  34. from oeqa.utils.commands import runCmd
  35. from oeqa.utils.metadata import metadata_from_bb, write_metadata_file
  36. # Set-up logging
  37. LOG_FORMAT = '[%(asctime)s] %(levelname)s: %(message)s'
  38. logging.basicConfig(level=logging.INFO, format=LOG_FORMAT,
  39. datefmt='%Y-%m-%d %H:%M:%S')
  40. log = logging.getLogger()
  41. def acquire_lock(lock_f):
  42. """Acquire flock on file"""
  43. log.debug("Acquiring lock %s", os.path.abspath(lock_f.name))
  44. try:
  45. fcntl.flock(lock_f, fcntl.LOCK_EX | fcntl.LOCK_NB)
  46. except IOError as err:
  47. if err.errno == errno.EAGAIN:
  48. return False
  49. raise
  50. log.debug("Lock acquired")
  51. return True
  52. def pre_run_sanity_check():
  53. """Sanity check of build environment"""
  54. build_dir = os.environ.get("BUILDDIR")
  55. if not build_dir:
  56. log.error("BUILDDIR not set. Please run the build environmnent setup "
  57. "script.")
  58. return False
  59. if os.getcwd() != build_dir:
  60. log.error("Please run this script under BUILDDIR (%s)", build_dir)
  61. return False
  62. ret = runCmd('which bitbake', ignore_status=True)
  63. if ret.status:
  64. log.error("bitbake command not found")
  65. return False
  66. return True
  67. def setup_file_logging(log_file):
  68. """Setup loggin to file"""
  69. log_dir = os.path.dirname(log_file)
  70. if not os.path.exists(log_dir):
  71. os.makedirs(log_dir)
  72. formatter = logging.Formatter(LOG_FORMAT)
  73. handler = logging.FileHandler(log_file)
  74. handler.setFormatter(formatter)
  75. log.addHandler(handler)
  76. def archive_build_conf(out_dir):
  77. """Archive build/conf to test results"""
  78. src_dir = os.path.join(os.environ['BUILDDIR'], 'conf')
  79. tgt_dir = os.path.join(out_dir, 'build', 'conf')
  80. os.makedirs(os.path.dirname(tgt_dir))
  81. shutil.copytree(src_dir, tgt_dir)
  82. def update_globalres_file(result_obj, filename, metadata):
  83. """Write results to globalres csv file"""
  84. # Map test names to time and size columns in globalres
  85. # The tuples represent index and length of times and sizes
  86. # respectively
  87. gr_map = {'test1': ((0, 1), (8, 1)),
  88. 'test12': ((1, 1), (None, None)),
  89. 'test13': ((2, 1), (9, 1)),
  90. 'test2': ((3, 1), (None, None)),
  91. 'test3': ((4, 3), (None, None)),
  92. 'test4': ((7, 1), (10, 2))}
  93. values = ['0'] * 12
  94. for status, test, _ in result_obj.all_results():
  95. if status in ['ERROR', 'SKIPPED']:
  96. continue
  97. (t_ind, t_len), (s_ind, s_len) = gr_map[test.name]
  98. if t_ind is not None:
  99. values[t_ind:t_ind + t_len] = test.times
  100. if s_ind is not None:
  101. values[s_ind:s_ind + s_len] = test.sizes
  102. log.debug("Writing globalres log to %s", filename)
  103. rev_info = metadata['layers']['meta']
  104. with open(filename, 'a') as fobj:
  105. fobj.write('{},{}:{},{},'.format(metadata['hostname'],
  106. rev_info['branch'],
  107. rev_info['commit'],
  108. rev_info['commit']))
  109. fobj.write(','.join(values) + '\n')
  110. def parse_args(argv):
  111. """Parse command line arguments"""
  112. parser = argparse.ArgumentParser(
  113. formatter_class=argparse.ArgumentDefaultsHelpFormatter)
  114. parser.add_argument('-D', '--debug', action='store_true',
  115. help='Enable debug level logging')
  116. parser.add_argument('--globalres-file',
  117. type=os.path.abspath,
  118. help="Append results to 'globalres' csv file")
  119. parser.add_argument('--lock-file', default='./oe-build-perf.lock',
  120. metavar='FILENAME', type=os.path.abspath,
  121. help="Lock file to use")
  122. parser.add_argument('-o', '--out-dir', default='results-{date}',
  123. type=os.path.abspath,
  124. help="Output directory for test results")
  125. parser.add_argument('-x', '--xml', action='store_true',
  126. help='Enable JUnit xml output')
  127. parser.add_argument('--log-file',
  128. default='{out_dir}/oe-build-perf-test.log',
  129. help="Log file of this script")
  130. parser.add_argument('--run-tests', nargs='+', metavar='TEST',
  131. help="List of tests to run")
  132. return parser.parse_args(argv)
  133. def main(argv=None):
  134. """Script entry point"""
  135. args = parse_args(argv)
  136. # Set-up log file
  137. out_dir = args.out_dir.format(date=datetime.now().strftime('%Y%m%d%H%M%S'))
  138. setup_file_logging(args.log_file.format(out_dir=out_dir))
  139. if args.debug:
  140. log.setLevel(logging.DEBUG)
  141. lock_f = open(args.lock_file, 'w')
  142. if not acquire_lock(lock_f):
  143. log.error("Another instance of this script is running, exiting...")
  144. return 1
  145. if not pre_run_sanity_check():
  146. return 1
  147. # Check our capability to drop caches and ask pass if needed
  148. KernelDropCaches.check()
  149. # Load build perf tests
  150. loader = BuildPerfTestLoader()
  151. if args.run_tests:
  152. suite = loader.loadTestsFromNames(args.run_tests, oeqa.buildperf)
  153. else:
  154. suite = loader.loadTestsFromModule(oeqa.buildperf)
  155. # Save test metadata
  156. metadata = metadata_from_bb()
  157. log.info("Testing Git revision branch:commit %s:%s (%s)",
  158. metadata['layers']['meta']['branch'],
  159. metadata['layers']['meta']['commit'],
  160. metadata['layers']['meta']['commit_count'])
  161. if args.xml:
  162. write_metadata_file(os.path.join(out_dir, 'metadata.xml'), metadata)
  163. else:
  164. with open(os.path.join(out_dir, 'metadata.json'), 'w') as fobj:
  165. json.dump(metadata, fobj, indent=2)
  166. archive_build_conf(out_dir)
  167. runner = BuildPerfTestRunner(out_dir, verbosity=2)
  168. # Suppress logger output to stderr so that the output from unittest
  169. # is not mixed with occasional logger output
  170. log.handlers[0].setLevel(logging.CRITICAL)
  171. # Run actual tests
  172. result = runner.run(suite)
  173. # Restore logger output to stderr
  174. log.handlers[0].setLevel(log.level)
  175. if args.xml:
  176. result.write_results_xml()
  177. else:
  178. result.write_results_json()
  179. result.write_buildstats_json()
  180. if args.globalres_file:
  181. update_globalres_file(result, args.globalres_file, metadata)
  182. if result.wasSuccessful():
  183. return 0
  184. return 2
  185. if __name__ == '__main__':
  186. sys.exit(main())