oe-selftest 9.6 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245
  1. #!/usr/bin/env python
  2. # Copyright (c) 2013 Intel Corporation
  3. #
  4. # This program is free software; you can redistribute it and/or modify
  5. # it under the terms of the GNU General Public License version 2 as
  6. # published by the Free Software Foundation.
  7. #
  8. # This program is distributed in the hope that it will be useful,
  9. # but WITHOUT ANY WARRANTY; without even the implied warranty of
  10. # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  11. # GNU General Public License for more details.
  12. #
  13. # You should have received a copy of the GNU General Public License along
  14. # with this program; if not, write to the Free Software Foundation, Inc.,
  15. # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
  16. # DESCRIPTION
  17. # This script runs tests defined in meta/lib/selftest/
  18. # It's purpose is to automate the testing of different bitbake tools.
  19. # To use it you just need to source your build environment setup script and
  20. # add the meta-selftest layer to your BBLAYERS.
  21. # Call the script as: "oe-selftest" to run all the tests in in meta/lib/selftest/
  22. # Call the script as: "oe-selftest <module>.<Class>.<method>" to run just a single test
  23. # E.g: "oe-selftest bboutput.BitbakeLayers" will run just the BitbakeLayers class from meta/lib/selftest/bboutput.py
  24. import os
  25. import sys
  26. import unittest
  27. import logging
  28. import argparse
  29. sys.path.insert(0, os.path.dirname(os.path.realpath(__file__)) + '/lib')
  30. import scriptpath
  31. scriptpath.add_bitbake_lib_path()
  32. scriptpath.add_oe_lib_path()
  33. import oeqa.selftest
  34. import oeqa.utils.ftools as ftools
  35. from oeqa.utils.commands import runCmd, get_bb_var, get_test_layer
  36. from oeqa.selftest.base import oeSelfTest
  37. def logger_create():
  38. log = logging.getLogger("selftest")
  39. log.setLevel(logging.DEBUG)
  40. fh = logging.FileHandler(filename='oe-selftest.log', mode='w')
  41. fh.setLevel(logging.DEBUG)
  42. ch = logging.StreamHandler(sys.stdout)
  43. ch.setLevel(logging.INFO)
  44. formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
  45. fh.setFormatter(formatter)
  46. ch.setFormatter(formatter)
  47. log.addHandler(fh)
  48. log.addHandler(ch)
  49. return log
  50. log = logger_create()
  51. def get_args_parser():
  52. description = "Script that runs unit tests agains bitbake and other Yocto related tools. The goal is to validate tools functionality and metadata integrity. Refer to https://wiki.yoctoproject.org/wiki/Oe-selftest for more information."
  53. parser = argparse.ArgumentParser(description=description)
  54. group = parser.add_mutually_exclusive_group(required=True)
  55. group.add_argument('--run-tests', required=False, action='store', nargs='*', dest="run_tests", default=None, help='Select what tests to run (modules, classes or test methods). Format should be: <module>.<class>.<test_method>')
  56. group.add_argument('--run-all-tests', required=False, action="store_true", dest="run_all_tests", default=False, help='Run all (unhidden) tests')
  57. group.add_argument('--list-modules', required=False, action="store_true", dest="list_modules", default=False, help='List all available test modules.')
  58. group.add_argument('--list-classes', required=False, action="store_true", dest="list_allclasses", default=False, help='List all available test classes.')
  59. return parser
  60. def preflight_check():
  61. log.info("Checking that everything is in order before running the tests")
  62. if not os.environ.get("BUILDDIR"):
  63. log.error("BUILDDIR isn't set. Did you forget to source your build environment setup script?")
  64. return False
  65. builddir = os.environ.get("BUILDDIR")
  66. if os.getcwd() != builddir:
  67. log.info("Changing cwd to %s" % builddir)
  68. os.chdir(builddir)
  69. if not "meta-selftest" in get_bb_var("BBLAYERS"):
  70. log.error("You don't seem to have the meta-selftest layer in BBLAYERS")
  71. return False
  72. log.info("Running bitbake -p")
  73. runCmd("bitbake -p")
  74. return True
  75. def add_include():
  76. builddir = os.environ.get("BUILDDIR")
  77. if "#include added by oe-selftest.py" \
  78. not in ftools.read_file(os.path.join(builddir, "conf/local.conf")):
  79. log.info("Adding: \"include selftest.inc\" in local.conf")
  80. ftools.append_file(os.path.join(builddir, "conf/local.conf"), \
  81. "\n#include added by oe-selftest.py\ninclude selftest.inc")
  82. if "#include added by oe-selftest.py" \
  83. not in ftools.read_file(os.path.join(builddir, "conf/bblayers.conf")):
  84. log.info("Adding: \"include bblayers.inc\" in bblayers.conf")
  85. ftools.append_file(os.path.join(builddir, "conf/bblayers.conf"), \
  86. "\n#include added by oe-selftest.py\ninclude bblayers.inc")
  87. def remove_include():
  88. builddir = os.environ.get("BUILDDIR")
  89. if builddir is None:
  90. return
  91. if "#include added by oe-selftest.py" \
  92. in ftools.read_file(os.path.join(builddir, "conf/local.conf")):
  93. log.info("Removing the include from local.conf")
  94. ftools.remove_from_file(os.path.join(builddir, "conf/local.conf"), \
  95. "#include added by oe-selftest.py\ninclude selftest.inc")
  96. if "#include added by oe-selftest.py" \
  97. in ftools.read_file(os.path.join(builddir, "conf/bblayers.conf")):
  98. log.info("Removing the include from bblayers.conf")
  99. ftools.remove_from_file(os.path.join(builddir, "conf/bblayers.conf"), \
  100. "#include added by oe-selftest.py\ninclude bblayers.inc")
  101. def remove_inc_files():
  102. try:
  103. os.remove(os.path.join(os.environ.get("BUILDDIR"), "conf/selftest.inc"))
  104. for root, _, files in os.walk(get_test_layer()):
  105. for f in files:
  106. if f == 'test_recipe.inc':
  107. os.remove(os.path.join(root, f))
  108. except (AttributeError, OSError,) as e: # AttributeError may happen if BUILDDIR is not set
  109. pass
  110. try:
  111. os.remove(os.path.join(os.environ.get("BUILDDIR"), "conf/bblayers.inc"))
  112. except:
  113. pass
  114. def get_tests(exclusive_modules=[], include_hidden=False):
  115. testslist = []
  116. for x in exclusive_modules:
  117. testslist.append('oeqa.selftest.' + x)
  118. if not testslist:
  119. for testpath in oeqa.selftest.__path__:
  120. files = sorted([f for f in os.listdir(testpath) if f.endswith('.py') and not (f.startswith('_') and not include_hidden) and not f.startswith('__') and f != 'base.py'])
  121. for f in files:
  122. module = 'oeqa.selftest.' + f[:-3]
  123. if module not in testslist:
  124. testslist.append(module)
  125. return testslist
  126. def main():
  127. parser = get_args_parser()
  128. args = parser.parse_args()
  129. # Add <layer>/lib to sys.path, so layers can add selftests
  130. log.info("Running bitbake -e to get BBPATH")
  131. bbpath = get_bb_var('BBPATH').split(':')
  132. layer_libdirs = [p for p in (os.path.join(l, 'lib') for l in bbpath) if os.path.exists(p)]
  133. sys.path.extend(layer_libdirs)
  134. reload(oeqa.selftest)
  135. if args.list_allclasses:
  136. args.list_modules = True
  137. if args.list_modules:
  138. log.info('Listing all available test modules:')
  139. testslist = get_tests(include_hidden=True)
  140. for test in testslist:
  141. module = test.split('.')[-1]
  142. info = ''
  143. if module.startswith('_'):
  144. info = ' (hidden)'
  145. print module + info
  146. if args.list_allclasses:
  147. try:
  148. import importlib
  149. modlib = importlib.import_module(test)
  150. for v in vars(modlib):
  151. t = vars(modlib)[v]
  152. if isinstance(t, type(oeSelfTest)) and issubclass(t, oeSelfTest) and t!=oeSelfTest:
  153. print " --", v
  154. for method in dir(t):
  155. if method.startswith("test_"):
  156. print " -- --", method
  157. except (AttributeError, ImportError) as e:
  158. print e
  159. pass
  160. if args.run_tests or args.run_all_tests:
  161. if not preflight_check():
  162. return 1
  163. testslist = get_tests(exclusive_modules=(args.run_tests or []), include_hidden=False)
  164. suite = unittest.TestSuite()
  165. loader = unittest.TestLoader()
  166. loader.sortTestMethodsUsing = None
  167. runner = unittest.TextTestRunner(verbosity=2, resultclass=StampedResult)
  168. # we need to do this here, otherwise just loading the tests
  169. # will take 2 minutes (bitbake -e calls)
  170. oeSelfTest.testlayer_path = get_test_layer()
  171. for test in testslist:
  172. log.info("Loading tests from: %s" % test)
  173. try:
  174. suite.addTests(loader.loadTestsFromName(test))
  175. except AttributeError as e:
  176. log.error("Failed to import %s" % test)
  177. log.error(e)
  178. return 1
  179. add_include()
  180. result = runner.run(suite)
  181. log.info("Finished")
  182. if result.wasSuccessful():
  183. return 0
  184. else:
  185. return 1
  186. class StampedResult(unittest.TextTestResult):
  187. """
  188. Custom TestResult that prints the time when a test starts. As oe-selftest
  189. can take a long time (ie a few hours) to run, timestamps help us understand
  190. what tests are taking a long time to execute.
  191. """
  192. def startTest(self, test):
  193. import time
  194. self.stream.write(time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()) + " - ")
  195. super(StampedResult, self).startTest(test)
  196. if __name__ == "__main__":
  197. try:
  198. ret = main()
  199. except Exception:
  200. ret = 1
  201. import traceback
  202. traceback.print_exc(5)
  203. finally:
  204. remove_include()
  205. remove_inc_files()
  206. sys.exit(ret)