oe-build-perf-test 7.0 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211
  1. #!/usr/bin/python3
  2. #
  3. # Build performance test script
  4. #
  5. # Copyright (c) 2016, Intel Corporation.
  6. #
  7. # This program is free software; you can redistribute it and/or modify it
  8. # under the terms and conditions of the GNU General Public License,
  9. # version 2, as published by the Free Software Foundation.
  10. #
  11. # This program is distributed in the hope it will be useful, but WITHOUT
  12. # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  13. # FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
  14. # more details.
  15. #
  16. """Build performance test script"""
  17. import argparse
  18. import errno
  19. import fcntl
  20. import logging
  21. import os
  22. import shutil
  23. import sys
  24. import unittest
  25. from datetime import datetime
  26. sys.path.insert(0, os.path.dirname(os.path.realpath(__file__)) + '/lib')
  27. import scriptpath
  28. scriptpath.add_oe_lib_path()
  29. import oeqa.buildperf
  30. from oeqa.buildperf import (BuildPerfTestLoader, BuildPerfTestResult,
  31. BuildPerfTestRunner, KernelDropCaches)
  32. from oeqa.utils.commands import runCmd
  33. from oeqa.utils.git import GitRepo, GitError
  34. # Set-up logging
  35. LOG_FORMAT = '[%(asctime)s] %(levelname)s: %(message)s'
  36. logging.basicConfig(level=logging.INFO, format=LOG_FORMAT,
  37. datefmt='%Y-%m-%d %H:%M:%S')
  38. log = logging.getLogger()
  39. def acquire_lock(lock_f):
  40. """Acquire flock on file"""
  41. log.debug("Acquiring lock %s", os.path.abspath(lock_f.name))
  42. try:
  43. fcntl.flock(lock_f, fcntl.LOCK_EX | fcntl.LOCK_NB)
  44. except IOError as err:
  45. if err.errno == errno.EAGAIN:
  46. return False
  47. raise
  48. log.debug("Lock acquired")
  49. return True
  50. def pre_run_sanity_check():
  51. """Sanity check of build environment"""
  52. build_dir = os.environ.get("BUILDDIR")
  53. if not build_dir:
  54. log.error("BUILDDIR not set. Please run the build environmnent setup "
  55. "script.")
  56. return False
  57. if os.getcwd() != build_dir:
  58. log.error("Please run this script under BUILDDIR (%s)", build_dir)
  59. return False
  60. ret = runCmd('which bitbake', ignore_status=True)
  61. if ret.status:
  62. log.error("bitbake command not found")
  63. return False
  64. return True
  65. def init_git_repo(path):
  66. """Check/create Git repository where to store results"""
  67. path = os.path.abspath(path)
  68. if os.path.isfile(path):
  69. log.error("Invalid Git repo %s: path exists but is not a directory", path)
  70. return False
  71. if not os.path.isdir(path):
  72. try:
  73. os.mkdir(path)
  74. except (FileNotFoundError, PermissionError) as err:
  75. log.error("Failed to mkdir %s: %s", path, err)
  76. return False
  77. if not os.listdir(path):
  78. log.info("Initializing a new Git repo at %s", path)
  79. GitRepo.init(path)
  80. try:
  81. GitRepo(path, is_topdir=True)
  82. except GitError:
  83. log.error("No Git repository but a non-empty directory found at %s.\n"
  84. "Please specify a Git repository, an empty directory or "
  85. "a non-existing directory", path)
  86. return False
  87. return True
  88. def setup_file_logging(log_file):
  89. """Setup loggin to file"""
  90. log_dir = os.path.dirname(log_file)
  91. if not os.path.exists(log_dir):
  92. os.makedirs(log_dir)
  93. formatter = logging.Formatter(LOG_FORMAT)
  94. handler = logging.FileHandler(log_file)
  95. handler.setFormatter(formatter)
  96. log.addHandler(handler)
  97. def archive_build_conf(out_dir):
  98. """Archive build/conf to test results"""
  99. src_dir = os.path.join(os.environ['BUILDDIR'], 'conf')
  100. tgt_dir = os.path.join(out_dir, 'build', 'conf')
  101. os.makedirs(os.path.dirname(tgt_dir))
  102. shutil.copytree(src_dir, tgt_dir)
  103. def parse_args(argv):
  104. """Parse command line arguments"""
  105. parser = argparse.ArgumentParser(
  106. formatter_class=argparse.ArgumentDefaultsHelpFormatter)
  107. parser.add_argument('-D', '--debug', action='store_true',
  108. help='Enable debug level logging')
  109. parser.add_argument('--globalres-file',
  110. type=os.path.abspath,
  111. help="Append results to 'globalres' csv file")
  112. parser.add_argument('--lock-file', default='./oe-build-perf.lock',
  113. metavar='FILENAME', type=os.path.abspath,
  114. help="Lock file to use")
  115. parser.add_argument('-o', '--out-dir', default='results-{date}',
  116. type=os.path.abspath,
  117. help="Output directory for test results")
  118. parser.add_argument('--log-file',
  119. default='{out_dir}/oe-build-perf-test.log',
  120. help="Log file of this script")
  121. parser.add_argument('--run-tests', nargs='+', metavar='TEST',
  122. help="List of tests to run")
  123. parser.add_argument('--commit-results', metavar='GIT_DIR',
  124. type=os.path.abspath,
  125. help="Commit result data to a (local) git repository")
  126. parser.add_argument('--commit-results-branch', metavar='BRANCH',
  127. default="{git_branch}",
  128. help="Commit results to branch BRANCH.")
  129. parser.add_argument('--commit-results-tag', metavar='TAG',
  130. default="{git_branch}/{git_commit_count}-g{git_commit}/{tag_num}",
  131. help="Tag results commit with TAG.")
  132. return parser.parse_args(argv)
  133. def main(argv=None):
  134. """Script entry point"""
  135. args = parse_args(argv)
  136. # Set-up log file
  137. out_dir = args.out_dir.format(date=datetime.now().strftime('%Y%m%d%H%M%S'))
  138. setup_file_logging(args.log_file.format(out_dir=out_dir))
  139. if args.debug:
  140. log.setLevel(logging.DEBUG)
  141. lock_f = open(args.lock_file, 'w')
  142. if not acquire_lock(lock_f):
  143. log.error("Another instance of this script is running, exiting...")
  144. return 1
  145. if not pre_run_sanity_check():
  146. return 1
  147. if args.commit_results:
  148. if not init_git_repo(args.commit_results):
  149. return 1
  150. # Check our capability to drop caches and ask pass if needed
  151. KernelDropCaches.check()
  152. # Load build perf tests
  153. loader = BuildPerfTestLoader()
  154. if args.run_tests:
  155. suite = loader.loadTestsFromNames(args.run_tests, oeqa.buildperf)
  156. else:
  157. suite = loader.loadTestsFromModule(oeqa.buildperf)
  158. archive_build_conf(out_dir)
  159. runner = BuildPerfTestRunner(out_dir, verbosity=2)
  160. # Suppress logger output to stderr so that the output from unittest
  161. # is not mixed with occasional logger output
  162. log.handlers[0].setLevel(logging.CRITICAL)
  163. # Run actual tests
  164. result = runner.run(suite)
  165. # Restore logger output to stderr
  166. log.handlers[0].setLevel(log.level)
  167. if args.globalres_file:
  168. result.update_globalres_file(args.globalres_file)
  169. if args.commit_results:
  170. result.git_commit_results(args.commit_results,
  171. args.commit_results_branch,
  172. args.commit_results_tag)
  173. if result.wasSuccessful():
  174. return 0
  175. return 2
  176. if __name__ == '__main__':
  177. sys.exit(main())