oe-selftest 33 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814
  1. #!/usr/bin/env python3
  2. # Copyright (c) 2013 Intel Corporation
  3. #
  4. # This program is free software; you can redistribute it and/or modify
  5. # it under the terms of the GNU General Public License version 2 as
  6. # published by the Free Software Foundation.
  7. #
  8. # This program is distributed in the hope that it will be useful,
  9. # but WITHOUT ANY WARRANTY; without even the implied warranty of
  10. # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  11. # GNU General Public License for more details.
  12. #
  13. # You should have received a copy of the GNU General Public License along
  14. # with this program; if not, write to the Free Software Foundation, Inc.,
  15. # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
  16. # DESCRIPTION
  17. # This script runs tests defined in meta/lib/oeqa/selftest/
  18. # It's purpose is to automate the testing of different bitbake tools.
  19. # To use it you just need to source your build environment setup script and
  20. # add the meta-selftest layer to your BBLAYERS.
  21. # Call the script as: "oe-selftest -a" to run all the tests in meta/lib/oeqa/selftest/
  22. # Call the script as: "oe-selftest -r <module>.<Class>.<method>" to run just a single test
  23. # E.g: "oe-selftest -r bblayers.BitbakeLayers" will run just the BitbakeLayers class from meta/lib/oeqa/selftest/bblayers.py
  24. import os
  25. import sys
  26. import unittest
  27. import logging
  28. import argparse
  29. import subprocess
  30. import time as t
  31. import re
  32. import fnmatch
  33. import collections
  34. import imp
  35. sys.path.insert(0, os.path.dirname(os.path.realpath(__file__)) + '/lib')
  36. import scriptpath
  37. scriptpath.add_bitbake_lib_path()
  38. scriptpath.add_oe_lib_path()
  39. import argparse_oe
  40. import oeqa.selftest
  41. import oeqa.utils.ftools as ftools
  42. from oeqa.utils.commands import runCmd, get_bb_var, get_test_layer
  43. from oeqa.utils.metadata import metadata_from_bb, write_metadata_file
  44. from oeqa.selftest.base import oeSelfTest, get_available_machines
  45. try:
  46. import xmlrunner
  47. from xmlrunner.result import _XMLTestResult as TestResult
  48. from xmlrunner import XMLTestRunner as _TestRunner
  49. except ImportError:
  50. # use the base runner instead
  51. from unittest import TextTestResult as TestResult
  52. from unittest import TextTestRunner as _TestRunner
  53. log_prefix = "oe-selftest-" + t.strftime("%Y%m%d-%H%M%S")
  54. def logger_create():
  55. log_file = log_prefix + ".log"
  56. if os.path.lexists("oe-selftest.log"):
  57. os.remove("oe-selftest.log")
  58. os.symlink(log_file, "oe-selftest.log")
  59. log = logging.getLogger("selftest")
  60. log.setLevel(logging.DEBUG)
  61. fh = logging.FileHandler(filename=log_file, mode='w')
  62. fh.setLevel(logging.DEBUG)
  63. ch = logging.StreamHandler(sys.stdout)
  64. ch.setLevel(logging.INFO)
  65. formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
  66. fh.setFormatter(formatter)
  67. ch.setFormatter(formatter)
  68. log.addHandler(fh)
  69. log.addHandler(ch)
  70. return log
  71. log = logger_create()
  72. def get_args_parser():
  73. description = "Script that runs unit tests against bitbake and other Yocto related tools. The goal is to validate tools functionality and metadata integrity. Refer to https://wiki.yoctoproject.org/wiki/Oe-selftest for more information."
  74. parser = argparse_oe.ArgumentParser(description=description)
  75. group = parser.add_mutually_exclusive_group(required=True)
  76. group.add_argument('-r', '--run-tests', required=False, action='store', nargs='*', dest="run_tests", default=None, help='Select what tests to run (modules, classes or test methods). Format should be: <module>.<class>.<test_method>')
  77. group.add_argument('-a', '--run-all-tests', required=False, action="store_true", dest="run_all_tests", default=False, help='Run all (unhidden) tests')
  78. group.add_argument('-m', '--list-modules', required=False, action="store_true", dest="list_modules", default=False, help='List all available test modules.')
  79. group.add_argument('--list-classes', required=False, action="store_true", dest="list_allclasses", default=False, help='List all available test classes.')
  80. parser.add_argument('--coverage', action="store_true", help="Run code coverage when testing")
  81. parser.add_argument('--coverage-source', dest="coverage_source", nargs="+", help="Specifiy the directories to take coverage from")
  82. parser.add_argument('--coverage-include', dest="coverage_include", nargs="+", help="Specify extra patterns to include into the coverage measurement")
  83. parser.add_argument('--coverage-omit', dest="coverage_omit", nargs="+", help="Specify with extra patterns to exclude from the coverage measurement")
  84. group.add_argument('--run-tests-by', required=False, dest='run_tests_by', default=False, nargs='*',
  85. help='run-tests-by <name|class|module|id|tag> <list of tests|classes|modules|ids|tags>')
  86. group.add_argument('--list-tests-by', required=False, dest='list_tests_by', default=False, nargs='*',
  87. help='list-tests-by <name|class|module|id|tag> <list of tests|classes|modules|ids|tags>')
  88. group.add_argument('-l', '--list-tests', required=False, action="store_true", dest="list_tests", default=False,
  89. help='List all available tests.')
  90. group.add_argument('--list-tags', required=False, dest='list_tags', default=False, action="store_true",
  91. help='List all tags that have been set to test cases.')
  92. parser.add_argument('--machine', required=False, dest='machine', choices=['random', 'all'], default=None,
  93. help='Run tests on different machines (random/all).')
  94. parser.add_argument('--repository', required=False, dest='repository', default='', action='store',
  95. help='Submit test results to a repository')
  96. return parser
  97. builddir = None
  98. def preflight_check():
  99. global builddir
  100. log.info("Checking that everything is in order before running the tests")
  101. if not os.environ.get("BUILDDIR"):
  102. log.error("BUILDDIR isn't set. Did you forget to source your build environment setup script?")
  103. return False
  104. builddir = os.environ.get("BUILDDIR")
  105. if os.getcwd() != builddir:
  106. log.info("Changing cwd to %s" % builddir)
  107. os.chdir(builddir)
  108. if not "meta-selftest" in get_bb_var("BBLAYERS"):
  109. log.warn("meta-selftest layer not found in BBLAYERS, adding it")
  110. meta_selftestdir = os.path.join(
  111. get_bb_var("BBLAYERS_FETCH_DIR"),
  112. 'meta-selftest')
  113. if os.path.isdir(meta_selftestdir):
  114. runCmd("bitbake-layers add-layer %s" %meta_selftestdir)
  115. else:
  116. log.error("could not locate meta-selftest in:\n%s"
  117. %meta_selftestdir)
  118. return False
  119. if "buildhistory.bbclass" in get_bb_var("BBINCLUDED"):
  120. log.error("You have buildhistory enabled already and this isn't recommended for selftest, please disable it first.")
  121. return False
  122. if get_bb_var("PRSERV_HOST"):
  123. log.error("Please unset PRSERV_HOST in order to run oe-selftest")
  124. return False
  125. if get_bb_var("SANITY_TESTED_DISTROS"):
  126. log.error("Please unset SANITY_TESTED_DISTROS in order to run oe-selftest")
  127. return False
  128. log.info("Running bitbake -p")
  129. runCmd("bitbake -p")
  130. return True
  131. def add_include():
  132. global builddir
  133. if "#include added by oe-selftest.py" \
  134. not in ftools.read_file(os.path.join(builddir, "conf/local.conf")):
  135. log.info("Adding: \"include selftest.inc\" in local.conf")
  136. ftools.append_file(os.path.join(builddir, "conf/local.conf"), \
  137. "\n#include added by oe-selftest.py\ninclude machine.inc\ninclude selftest.inc")
  138. if "#include added by oe-selftest.py" \
  139. not in ftools.read_file(os.path.join(builddir, "conf/bblayers.conf")):
  140. log.info("Adding: \"include bblayers.inc\" in bblayers.conf")
  141. ftools.append_file(os.path.join(builddir, "conf/bblayers.conf"), \
  142. "\n#include added by oe-selftest.py\ninclude bblayers.inc")
  143. def remove_include():
  144. global builddir
  145. if builddir is None:
  146. return
  147. if "#include added by oe-selftest.py" \
  148. in ftools.read_file(os.path.join(builddir, "conf/local.conf")):
  149. log.info("Removing the include from local.conf")
  150. ftools.remove_from_file(os.path.join(builddir, "conf/local.conf"), \
  151. "\n#include added by oe-selftest.py\ninclude machine.inc\ninclude selftest.inc")
  152. if "#include added by oe-selftest.py" \
  153. in ftools.read_file(os.path.join(builddir, "conf/bblayers.conf")):
  154. log.info("Removing the include from bblayers.conf")
  155. ftools.remove_from_file(os.path.join(builddir, "conf/bblayers.conf"), \
  156. "\n#include added by oe-selftest.py\ninclude bblayers.inc")
  157. def remove_inc_files():
  158. global builddir
  159. if builddir is None:
  160. return
  161. try:
  162. os.remove(os.path.join(builddir, "conf/selftest.inc"))
  163. for root, _, files in os.walk(get_test_layer()):
  164. for f in files:
  165. if f == 'test_recipe.inc':
  166. os.remove(os.path.join(root, f))
  167. except OSError as e:
  168. pass
  169. for incl_file in ['conf/bblayers.inc', 'conf/machine.inc']:
  170. try:
  171. os.remove(os.path.join(builddir, incl_file))
  172. except:
  173. pass
  174. def get_tests_modules(include_hidden=False):
  175. modules_list = list()
  176. for modules_path in oeqa.selftest.__path__:
  177. for (p, d, f) in os.walk(modules_path):
  178. files = sorted([f for f in os.listdir(p) if f.endswith('.py') and not (f.startswith('_') and not include_hidden) and not f.startswith('__') and f != 'base.py'])
  179. for f in files:
  180. submodules = p.split("selftest")[-1]
  181. module = ""
  182. if submodules:
  183. module = 'oeqa.selftest' + submodules.replace("/",".") + "." + f.split('.py')[0]
  184. else:
  185. module = 'oeqa.selftest.' + f.split('.py')[0]
  186. if module not in modules_list:
  187. modules_list.append(module)
  188. return modules_list
  189. def get_tests(exclusive_modules=[], include_hidden=False):
  190. test_modules = list()
  191. for x in exclusive_modules:
  192. test_modules.append('oeqa.selftest.' + x)
  193. if not test_modules:
  194. inc_hidden = include_hidden
  195. test_modules = get_tests_modules(inc_hidden)
  196. return test_modules
  197. class Tc:
  198. def __init__(self, tcname, tcclass, tcmodule, tcid=None, tctag=None):
  199. self.tcname = tcname
  200. self.tcclass = tcclass
  201. self.tcmodule = tcmodule
  202. self.tcid = tcid
  203. # A test case can have multiple tags (as tuples) otherwise str will suffice
  204. self.tctag = tctag
  205. self.fullpath = '.'.join(['oeqa', 'selftest', tcmodule, tcclass, tcname])
  206. def get_tests_from_module(tmod):
  207. tlist = []
  208. prefix = 'oeqa.selftest.'
  209. try:
  210. import importlib
  211. modlib = importlib.import_module(tmod)
  212. for mod in list(vars(modlib).values()):
  213. if isinstance(mod, type(oeSelfTest)) and issubclass(mod, oeSelfTest) and mod is not oeSelfTest:
  214. for test in dir(mod):
  215. if test.startswith('test_') and hasattr(vars(mod)[test], '__call__'):
  216. # Get test case id and feature tag
  217. # NOTE: if testcase decorator or feature tag not set will throw error
  218. try:
  219. tid = vars(mod)[test].test_case
  220. except:
  221. print('DEBUG: tc id missing for ' + str(test))
  222. tid = None
  223. try:
  224. ttag = vars(mod)[test].tag__feature
  225. except:
  226. # print('DEBUG: feature tag missing for ' + str(test))
  227. ttag = None
  228. # NOTE: for some reason lstrip() doesn't work for mod.__module__
  229. tlist.append(Tc(test, mod.__name__, mod.__module__.replace(prefix, ''), tid, ttag))
  230. except:
  231. pass
  232. return tlist
  233. def get_all_tests():
  234. # Get all the test modules (except the hidden ones)
  235. testlist = []
  236. tests_modules = get_tests_modules()
  237. # Get all the tests from modules
  238. for tmod in sorted(tests_modules):
  239. testlist += get_tests_from_module(tmod)
  240. return testlist
  241. def get_testsuite_by(criteria, keyword):
  242. # Get a testsuite based on 'keyword'
  243. # criteria: name, class, module, id, tag
  244. # keyword: a list of tests, classes, modules, ids, tags
  245. ts = []
  246. all_tests = get_all_tests()
  247. def get_matches(values):
  248. # Get an item and return the ones that match with keyword(s)
  249. # values: the list of items (names, modules, classes...)
  250. result = []
  251. remaining = values[:]
  252. for key in keyword:
  253. found = False
  254. if key in remaining:
  255. # Regular matching of exact item
  256. result.append(key)
  257. remaining.remove(key)
  258. found = True
  259. else:
  260. # Wildcard matching
  261. pattern = re.compile(fnmatch.translate(r"%s" % key))
  262. added = [x for x in remaining if pattern.match(x)]
  263. if added:
  264. result.extend(added)
  265. remaining = [x for x in remaining if x not in added]
  266. found = True
  267. if not found:
  268. log.error("Failed to find test: %s" % key)
  269. return result
  270. if criteria == 'name':
  271. names = get_matches([ tc.tcname for tc in all_tests ])
  272. ts = [ tc for tc in all_tests if tc.tcname in names ]
  273. elif criteria == 'class':
  274. classes = get_matches([ tc.tcclass for tc in all_tests ])
  275. ts = [ tc for tc in all_tests if tc.tcclass in classes ]
  276. elif criteria == 'module':
  277. modules = get_matches([ tc.tcmodule for tc in all_tests ])
  278. ts = [ tc for tc in all_tests if tc.tcmodule in modules ]
  279. elif criteria == 'id':
  280. ids = get_matches([ str(tc.tcid) for tc in all_tests ])
  281. ts = [ tc for tc in all_tests if str(tc.tcid) in ids ]
  282. elif criteria == 'tag':
  283. values = set()
  284. for tc in all_tests:
  285. # tc can have multiple tags (as tuple) otherwise str will suffice
  286. if isinstance(tc.tctag, tuple):
  287. values |= { str(tag) for tag in tc.tctag }
  288. else:
  289. values.add(str(tc.tctag))
  290. tags = get_matches(list(values))
  291. for tc in all_tests:
  292. for tag in tags:
  293. if isinstance(tc.tctag, tuple) and tag in tc.tctag:
  294. ts.append(tc)
  295. elif tag == tc.tctag:
  296. ts.append(tc)
  297. # Remove duplicates from the list
  298. ts = list(set(ts))
  299. return ts
  300. def list_testsuite_by(criteria, keyword):
  301. # Get a testsuite based on 'keyword'
  302. # criteria: name, class, module, id, tag
  303. # keyword: a list of tests, classes, modules, ids, tags
  304. def tc_key(t):
  305. if t[0] is None:
  306. return (0,) + t[1:]
  307. return t
  308. # tcid may be None if no ID was assigned, in which case sorted() will throw
  309. # a TypeError as Python 3 does not allow comparison (<,<=,>=,>) of
  310. # heterogeneous types, handle this by using a custom key generator
  311. ts = sorted([ (tc.tcid, tc.tctag, tc.tcname, tc.tcclass, tc.tcmodule) \
  312. for tc in get_testsuite_by(criteria, keyword) ], key=tc_key)
  313. print('_' * 150)
  314. for t in ts:
  315. if isinstance(t[1], (tuple, list)):
  316. print('%-4s\t%-20s\t%-60s\t%-25s\t%-20s' % (t[0], ', '.join(t[1]), t[2], t[3], t[4]))
  317. else:
  318. print('%-4s\t%-20s\t%-60s\t%-25s\t%-20s' % t)
  319. print('_' * 150)
  320. print('Filtering by:\t %s' % criteria)
  321. print('Looking for:\t %s' % ', '.join(str(x) for x in keyword))
  322. print('Total found:\t %s' % len(ts))
  323. def list_tests():
  324. # List all available oe-selftest tests
  325. ts = get_all_tests()
  326. print('%-4s\t%-10s\t%-50s' % ('id', 'tag', 'test'))
  327. print('_' * 80)
  328. for t in ts:
  329. if isinstance(t.tctag, (tuple, list)):
  330. print('%-4s\t%-10s\t%-50s' % (t.tcid, ', '.join(t.tctag), '.'.join([t.tcmodule, t.tcclass, t.tcname])))
  331. else:
  332. print('%-4s\t%-10s\t%-50s' % (t.tcid, t.tctag, '.'.join([t.tcmodule, t.tcclass, t.tcname])))
  333. print('_' * 80)
  334. print('Total found:\t %s' % len(ts))
  335. def list_tags():
  336. # Get all tags set to test cases
  337. # This is useful when setting tags to test cases
  338. # The list of tags should be kept as minimal as possible
  339. tags = set()
  340. all_tests = get_all_tests()
  341. for tc in all_tests:
  342. if isinstance(tc.tctag, (tuple, list)):
  343. tags.update(set(tc.tctag))
  344. else:
  345. tags.add(tc.tctag)
  346. print('Tags:\t%s' % ', '.join(str(x) for x in tags))
  347. def coverage_setup(coverage_source, coverage_include, coverage_omit):
  348. """ Set up the coverage measurement for the testcases to be run """
  349. import datetime
  350. import subprocess
  351. global builddir
  352. pokydir = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
  353. curcommit= subprocess.check_output(["git", "--git-dir", os.path.join(pokydir, ".git"), "rev-parse", "HEAD"]).decode('utf-8')
  354. coveragerc = "%s/.coveragerc" % builddir
  355. data_file = "%s/.coverage." % builddir
  356. data_file += datetime.datetime.now().strftime('%Y%m%dT%H%M%S')
  357. if os.path.isfile(data_file):
  358. os.remove(data_file)
  359. with open(coveragerc, 'w') as cps:
  360. cps.write("# Generated with command '%s'\n" % " ".join(sys.argv))
  361. cps.write("# HEAD commit %s\n" % curcommit.strip())
  362. cps.write("[run]\n")
  363. cps.write("data_file = %s\n" % data_file)
  364. cps.write("branch = True\n")
  365. # Measure just BBLAYERS, scripts and bitbake folders
  366. cps.write("source = \n")
  367. if coverage_source:
  368. for directory in coverage_source:
  369. if not os.path.isdir(directory):
  370. log.warn("Directory %s is not valid.", directory)
  371. cps.write(" %s\n" % directory)
  372. else:
  373. for layer in get_bb_var('BBLAYERS').split():
  374. cps.write(" %s\n" % layer)
  375. cps.write(" %s\n" % os.path.dirname(os.path.realpath(__file__)))
  376. cps.write(" %s\n" % os.path.join(os.path.dirname(os.path.dirname(os.path.realpath(__file__))),'bitbake'))
  377. if coverage_include:
  378. cps.write("include = \n")
  379. for pattern in coverage_include:
  380. cps.write(" %s\n" % pattern)
  381. if coverage_omit:
  382. cps.write("omit = \n")
  383. for pattern in coverage_omit:
  384. cps.write(" %s\n" % pattern)
  385. return coveragerc
  386. def coverage_report():
  387. """ Loads the coverage data gathered and reports it back """
  388. try:
  389. # Coverage4 uses coverage.Coverage
  390. from coverage import Coverage
  391. except:
  392. # Coverage under version 4 uses coverage.coverage
  393. from coverage import coverage as Coverage
  394. import io as StringIO
  395. from coverage.misc import CoverageException
  396. cov_output = StringIO.StringIO()
  397. # Creating the coverage data with the setting from the configuration file
  398. cov = Coverage(config_file = os.environ.get('COVERAGE_PROCESS_START'))
  399. try:
  400. # Load data from the data file specified in the configuration
  401. cov.load()
  402. # Store report data in a StringIO variable
  403. cov.report(file = cov_output, show_missing=False)
  404. log.info("\n%s" % cov_output.getvalue())
  405. except CoverageException as e:
  406. # Show problems with the reporting. Since Coverage4 not finding any data to report raises an exception
  407. log.warn("%s" % str(e))
  408. finally:
  409. cov_output.close()
  410. def main():
  411. parser = get_args_parser()
  412. args = parser.parse_args()
  413. # Add <layer>/lib to sys.path, so layers can add selftests
  414. log.info("Running bitbake -e to get BBPATH")
  415. bbpath = get_bb_var('BBPATH').split(':')
  416. layer_libdirs = [p for p in (os.path.join(l, 'lib') for l in bbpath) if os.path.exists(p)]
  417. sys.path.extend(layer_libdirs)
  418. imp.reload(oeqa.selftest)
  419. # act like bitbake and enforce en_US.UTF-8 locale
  420. os.environ["LC_ALL"] = "en_US.UTF-8"
  421. if args.run_tests_by and len(args.run_tests_by) >= 2:
  422. valid_options = ['name', 'class', 'module', 'id', 'tag']
  423. if args.run_tests_by[0] not in valid_options:
  424. print('--run-tests-by %s not a valid option. Choose one of <name|class|module|id|tag>.' % args.run_tests_by[0])
  425. return 1
  426. else:
  427. criteria = args.run_tests_by[0]
  428. keyword = args.run_tests_by[1:]
  429. ts = sorted([ tc.fullpath for tc in get_testsuite_by(criteria, keyword) ])
  430. if not ts:
  431. return 1
  432. if args.list_tests_by and len(args.list_tests_by) >= 2:
  433. valid_options = ['name', 'class', 'module', 'id', 'tag']
  434. if args.list_tests_by[0] not in valid_options:
  435. print('--list-tests-by %s not a valid option. Choose one of <name|class|module|id|tag>.' % args.list_tests_by[0])
  436. return 1
  437. else:
  438. criteria = args.list_tests_by[0]
  439. keyword = args.list_tests_by[1:]
  440. list_testsuite_by(criteria, keyword)
  441. if args.list_tests:
  442. list_tests()
  443. if args.list_tags:
  444. list_tags()
  445. if args.list_allclasses:
  446. args.list_modules = True
  447. if args.list_modules:
  448. log.info('Listing all available test modules:')
  449. testslist = get_tests(include_hidden=True)
  450. for test in testslist:
  451. module = test.split('oeqa.selftest.')[-1]
  452. info = ''
  453. if module.startswith('_'):
  454. info = ' (hidden)'
  455. print(module + info)
  456. if args.list_allclasses:
  457. try:
  458. import importlib
  459. modlib = importlib.import_module(test)
  460. for v in vars(modlib):
  461. t = vars(modlib)[v]
  462. if isinstance(t, type(oeSelfTest)) and issubclass(t, oeSelfTest) and t!=oeSelfTest:
  463. print(" --", v)
  464. for method in dir(t):
  465. if method.startswith("test_") and isinstance(vars(t)[method], collections.Callable):
  466. print(" -- --", method)
  467. except (AttributeError, ImportError) as e:
  468. print(e)
  469. pass
  470. if args.run_tests or args.run_all_tests or args.run_tests_by:
  471. if not preflight_check():
  472. return 1
  473. if args.run_tests_by:
  474. testslist = ts
  475. else:
  476. testslist = get_tests(exclusive_modules=(args.run_tests or []), include_hidden=False)
  477. suite = unittest.TestSuite()
  478. loader = unittest.TestLoader()
  479. loader.sortTestMethodsUsing = None
  480. runner = TestRunner(verbosity=2,
  481. resultclass=buildResultClass(args))
  482. # we need to do this here, otherwise just loading the tests
  483. # will take 2 minutes (bitbake -e calls)
  484. oeSelfTest.testlayer_path = get_test_layer()
  485. for test in testslist:
  486. log.info("Loading tests from: %s" % test)
  487. try:
  488. suite.addTests(loader.loadTestsFromName(test))
  489. except AttributeError as e:
  490. log.error("Failed to import %s" % test)
  491. log.error(e)
  492. return 1
  493. add_include()
  494. if args.machine:
  495. # Custom machine sets only weak default values (??=) for MACHINE in machine.inc
  496. # This let test cases that require a specific MACHINE to be able to override it, using (?= or =)
  497. log.info('Custom machine mode enabled. MACHINE set to %s' % args.machine)
  498. if args.machine == 'random':
  499. os.environ['CUSTOMMACHINE'] = 'random'
  500. result = runner.run(suite)
  501. else: # all
  502. machines = get_available_machines()
  503. for m in machines:
  504. log.info('Run tests with custom MACHINE set to: %s' % m)
  505. os.environ['CUSTOMMACHINE'] = m
  506. result = runner.run(suite)
  507. else:
  508. result = runner.run(suite)
  509. log.info("Finished")
  510. if args.repository:
  511. import git
  512. # Commit tests results to repository
  513. metadata = metadata_from_bb()
  514. git_dir = os.path.join(os.getcwd(), 'selftest')
  515. if not os.path.isdir(git_dir):
  516. os.mkdir(git_dir)
  517. log.debug('Checking for git repository in %s' % git_dir)
  518. try:
  519. repo = git.Repo(git_dir)
  520. except git.exc.InvalidGitRepositoryError:
  521. log.debug("Couldn't find git repository %s; "
  522. "cloning from %s" % (git_dir, args.repository))
  523. repo = git.Repo.clone_from(args.repository, git_dir)
  524. r_branches = repo.git.branch(r=True)
  525. r_branches = set(r_branches.replace('origin/', '').split())
  526. l_branches = {str(branch) for branch in repo.branches}
  527. branch = '%s/%s/%s' % (metadata['hostname'],
  528. metadata['layers']['meta'].get('branch', '(nogit)'),
  529. metadata['config']['MACHINE'])
  530. if branch in l_branches:
  531. log.debug('Found branch in local repository, checking out')
  532. repo.git.checkout(branch)
  533. elif branch in r_branches:
  534. log.debug('Found branch in remote repository, checking'
  535. ' out and pulling')
  536. repo.git.checkout(branch)
  537. repo.git.pull()
  538. else:
  539. log.debug('New branch %s' % branch)
  540. repo.git.checkout('master')
  541. repo.git.checkout(b=branch)
  542. cleanResultsDir(repo)
  543. xml_dir = os.path.join(os.getcwd(), log_prefix)
  544. copyResultFiles(xml_dir, git_dir, repo)
  545. metadata_file = os.path.join(git_dir, 'metadata.xml')
  546. write_metadata_file(metadata_file, metadata)
  547. repo.index.add([metadata_file])
  548. repo.index.write()
  549. # Get information for commit message
  550. layer_info = ''
  551. for layer, values in metadata['layers'].items():
  552. layer_info = '%s%-17s = %s:%s\n' % (layer_info, layer,
  553. values.get('branch', '(nogit)'), values.get('commit', '0'*40))
  554. msg = 'Selftest for build %s of %s for machine %s on %s\n\n%s' % (
  555. log_prefix[12:], metadata['distro']['pretty_name'],
  556. metadata['config']['MACHINE'], metadata['hostname'], layer_info)
  557. log.debug('Commiting results to local repository')
  558. repo.index.commit(msg)
  559. if not repo.is_dirty():
  560. try:
  561. if branch in r_branches:
  562. log.debug('Pushing changes to remote repository')
  563. repo.git.push()
  564. else:
  565. log.debug('Pushing changes to remote repository '
  566. 'creating new branch')
  567. repo.git.push('-u', 'origin', branch)
  568. except GitCommandError:
  569. log.error('Falied to push to remote repository')
  570. return 1
  571. else:
  572. log.error('Local repository is dirty, not pushing commits')
  573. if result.wasSuccessful():
  574. return 0
  575. else:
  576. return 1
  577. def buildResultClass(args):
  578. """Build a Result Class to use in the testcase execution"""
  579. import site
  580. class StampedResult(TestResult):
  581. """
  582. Custom TestResult that prints the time when a test starts. As oe-selftest
  583. can take a long time (ie a few hours) to run, timestamps help us understand
  584. what tests are taking a long time to execute.
  585. If coverage is required, this class executes the coverage setup and reporting.
  586. """
  587. def startTest(self, test):
  588. import time
  589. self.stream.write(time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()) + " - ")
  590. super(StampedResult, self).startTest(test)
  591. def startTestRun(self):
  592. """ Setup coverage before running any testcase """
  593. # variable holding the coverage configuration file allowing subprocess to be measured
  594. self.coveragepth = None
  595. # indicates the system if coverage is currently installed
  596. self.coverage_installed = True
  597. if args.coverage or args.coverage_source or args.coverage_include or args.coverage_omit:
  598. try:
  599. # check if user can do coverage
  600. import coverage
  601. except:
  602. log.warn("python coverage is not installed. More info on https://pypi.python.org/pypi/coverage")
  603. self.coverage_installed = False
  604. if self.coverage_installed:
  605. log.info("Coverage is enabled")
  606. major_version = int(coverage.version.__version__[0])
  607. if major_version < 4:
  608. log.error("python coverage %s installed. Require version 4 or greater." % coverage.version.__version__)
  609. self.stop()
  610. # In case the user has not set the variable COVERAGE_PROCESS_START,
  611. # create a default one and export it. The COVERAGE_PROCESS_START
  612. # value indicates where the coverage configuration file resides
  613. # More info on https://pypi.python.org/pypi/coverage
  614. if not os.environ.get('COVERAGE_PROCESS_START'):
  615. os.environ['COVERAGE_PROCESS_START'] = coverage_setup(args.coverage_source, args.coverage_include, args.coverage_omit)
  616. # Use default site.USER_SITE and write corresponding config file
  617. site.ENABLE_USER_SITE = True
  618. if not os.path.exists(site.USER_SITE):
  619. os.makedirs(site.USER_SITE)
  620. self.coveragepth = os.path.join(site.USER_SITE, "coverage.pth")
  621. with open(self.coveragepth, 'w') as cps:
  622. cps.write('import sys,site; sys.path.extend(site.getsitepackages()); import coverage; coverage.process_startup();')
  623. def stopTestRun(self):
  624. """ Report coverage data after the testcases are run """
  625. if args.coverage or args.coverage_source or args.coverage_include or args.coverage_omit:
  626. if self.coverage_installed:
  627. with open(os.environ['COVERAGE_PROCESS_START']) as ccf:
  628. log.info("Coverage configuration file (%s)" % os.environ.get('COVERAGE_PROCESS_START'))
  629. log.info("===========================")
  630. log.info("\n%s" % "".join(ccf.readlines()))
  631. log.info("Coverage Report")
  632. log.info("===============")
  633. try:
  634. coverage_report()
  635. finally:
  636. # remove the pth file
  637. try:
  638. os.remove(self.coveragepth)
  639. except OSError:
  640. log.warn("Expected temporal file from coverage is missing, ignoring removal.")
  641. return StampedResult
  642. def cleanResultsDir(repo):
  643. """ Remove result files from directory """
  644. xml_files = []
  645. directory = repo.working_tree_dir
  646. for f in os.listdir(directory):
  647. path = os.path.join(directory, f)
  648. if os.path.isfile(path) and path.endswith('.xml'):
  649. xml_files.append(f)
  650. repo.index.remove(xml_files, working_tree=True)
  651. def copyResultFiles(src, dst, repo):
  652. """ Copy result files from src to dst removing the time stamp. """
  653. import shutil
  654. re_time = re.compile("-[0-9]+")
  655. file_list = []
  656. for root, subdirs, files in os.walk(src):
  657. tmp_dir = root.replace(src, '').lstrip('/')
  658. for s in subdirs:
  659. os.mkdir(os.path.join(dst, tmp_dir, s))
  660. for f in files:
  661. file_name = os.path.join(dst, tmp_dir, re_time.sub("", f))
  662. shutil.copy2(os.path.join(root, f), file_name)
  663. file_list.append(file_name)
  664. repo.index.add(file_list)
  665. class TestRunner(_TestRunner):
  666. """Test runner class aware of exporting tests."""
  667. def __init__(self, *args, **kwargs):
  668. try:
  669. exportdir = os.path.join(os.getcwd(), log_prefix)
  670. kwargsx = dict(**kwargs)
  671. # argument specific to XMLTestRunner, if adding a new runner then
  672. # also add logic to use other runner's args.
  673. kwargsx['output'] = exportdir
  674. kwargsx['descriptions'] = False
  675. # done for the case where telling the runner where to export
  676. super(TestRunner, self).__init__(*args, **kwargsx)
  677. except TypeError:
  678. log.info("test runner init'ed like unittest")
  679. super(TestRunner, self).__init__(*args, **kwargs)
  680. if __name__ == "__main__":
  681. try:
  682. ret = main()
  683. except Exception:
  684. ret = 1
  685. import traceback
  686. traceback.print_exc()
  687. finally:
  688. remove_include()
  689. remove_inc_files()
  690. sys.exit(ret)