oe-selftest 27 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676
  1. #!/usr/bin/env python3
  2. # Copyright (c) 2013 Intel Corporation
  3. #
  4. # This program is free software; you can redistribute it and/or modify
  5. # it under the terms of the GNU General Public License version 2 as
  6. # published by the Free Software Foundation.
  7. #
  8. # This program is distributed in the hope that it will be useful,
  9. # but WITHOUT ANY WARRANTY; without even the implied warranty of
  10. # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  11. # GNU General Public License for more details.
  12. #
  13. # You should have received a copy of the GNU General Public License along
  14. # with this program; if not, write to the Free Software Foundation, Inc.,
  15. # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
  16. # DESCRIPTION
  17. # This script runs tests defined in meta/lib/oeqa/selftest/
  18. # It's purpose is to automate the testing of different bitbake tools.
  19. # To use it you just need to source your build environment setup script and
  20. # add the meta-selftest layer to your BBLAYERS.
  21. # Call the script as: "oe-selftest -a" to run all the tests in meta/lib/oeqa/selftest/
  22. # Call the script as: "oe-selftest -r <module>.<Class>.<method>" to run just a single test
  23. # E.g: "oe-selftest -r bblayers.BitbakeLayers" will run just the BitbakeLayers class from meta/lib/oeqa/selftest/bblayers.py
  24. import os
  25. import sys
  26. import unittest
  27. import logging
  28. import argparse
  29. import subprocess
  30. import time as t
  31. import re
  32. import fnmatch
  33. import collections
  34. import imp
  35. sys.path.insert(0, os.path.dirname(os.path.realpath(__file__)) + '/lib')
  36. import scriptpath
  37. scriptpath.add_bitbake_lib_path()
  38. scriptpath.add_oe_lib_path()
  39. import argparse_oe
  40. import oeqa.selftest
  41. import oeqa.utils.ftools as ftools
  42. from oeqa.utils.commands import runCmd, get_bb_var, get_test_layer
  43. from oeqa.selftest.base import oeSelfTest, get_available_machines
  44. try:
  45. import xmlrunner
  46. from xmlrunner.result import _XMLTestResult as TestResult
  47. from xmlrunner import XMLTestRunner as _TestRunner
  48. except ImportError:
  49. # use the base runner instead
  50. from unittest import TextTestResult as TestResult
  51. from unittest import TextTestRunner as _TestRunner
  52. log_prefix = "oe-selftest-" + t.strftime("%Y%m%d-%H%M%S")
  53. def logger_create():
  54. log_file = log_prefix + ".log"
  55. if os.path.exists("oe-selftest.log"): os.remove("oe-selftest.log")
  56. os.symlink(log_file, "oe-selftest.log")
  57. log = logging.getLogger("selftest")
  58. log.setLevel(logging.DEBUG)
  59. fh = logging.FileHandler(filename=log_file, mode='w')
  60. fh.setLevel(logging.DEBUG)
  61. ch = logging.StreamHandler(sys.stdout)
  62. ch.setLevel(logging.INFO)
  63. formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
  64. fh.setFormatter(formatter)
  65. ch.setFormatter(formatter)
  66. log.addHandler(fh)
  67. log.addHandler(ch)
  68. return log
  69. log = logger_create()
  70. def get_args_parser():
  71. description = "Script that runs unit tests agains bitbake and other Yocto related tools. The goal is to validate tools functionality and metadata integrity. Refer to https://wiki.yoctoproject.org/wiki/Oe-selftest for more information."
  72. parser = argparse_oe.ArgumentParser(description=description)
  73. group = parser.add_mutually_exclusive_group(required=True)
  74. group.add_argument('-r', '--run-tests', required=False, action='store', nargs='*', dest="run_tests", default=None, help='Select what tests to run (modules, classes or test methods). Format should be: <module>.<class>.<test_method>')
  75. group.add_argument('-a', '--run-all-tests', required=False, action="store_true", dest="run_all_tests", default=False, help='Run all (unhidden) tests')
  76. group.add_argument('-m', '--list-modules', required=False, action="store_true", dest="list_modules", default=False, help='List all available test modules.')
  77. group.add_argument('--list-classes', required=False, action="store_true", dest="list_allclasses", default=False, help='List all available test classes.')
  78. parser.add_argument('--coverage', action="store_true", help="Run code coverage when testing")
  79. parser.add_argument('--coverage-source', dest="coverage_source", nargs="+", help="Specifiy the directories to take coverage from")
  80. parser.add_argument('--coverage-include', dest="coverage_include", nargs="+", help="Specify extra patterns to include into the coverage measurement")
  81. parser.add_argument('--coverage-omit', dest="coverage_omit", nargs="+", help="Specify with extra patterns to exclude from the coverage measurement")
  82. group.add_argument('--run-tests-by', required=False, dest='run_tests_by', default=False, nargs='*',
  83. help='run-tests-by <name|class|module|id|tag> <list of tests|classes|modules|ids|tags>')
  84. group.add_argument('--list-tests-by', required=False, dest='list_tests_by', default=False, nargs='*',
  85. help='list-tests-by <name|class|module|id|tag> <list of tests|classes|modules|ids|tags>')
  86. group.add_argument('-l', '--list-tests', required=False, action="store_true", dest="list_tests", default=False,
  87. help='List all available tests.')
  88. group.add_argument('--list-tags', required=False, dest='list_tags', default=False, action="store_true",
  89. help='List all tags that have been set to test cases.')
  90. parser.add_argument('--machine', required=False, dest='machine', choices=['random', 'all'], default=None,
  91. help='Run tests on different machines (random/all).')
  92. return parser
  93. def preflight_check():
  94. log.info("Checking that everything is in order before running the tests")
  95. if not os.environ.get("BUILDDIR"):
  96. log.error("BUILDDIR isn't set. Did you forget to source your build environment setup script?")
  97. return False
  98. builddir = os.environ.get("BUILDDIR")
  99. if os.getcwd() != builddir:
  100. log.info("Changing cwd to %s" % builddir)
  101. os.chdir(builddir)
  102. if not "meta-selftest" in get_bb_var("BBLAYERS"):
  103. log.error("You don't seem to have the meta-selftest layer in BBLAYERS")
  104. return False
  105. log.info("Running bitbake -p")
  106. runCmd("bitbake -p")
  107. return True
  108. def add_include():
  109. builddir = os.environ.get("BUILDDIR")
  110. if "#include added by oe-selftest.py" \
  111. not in ftools.read_file(os.path.join(builddir, "conf/local.conf")):
  112. log.info("Adding: \"include selftest.inc\" in local.conf")
  113. ftools.append_file(os.path.join(builddir, "conf/local.conf"), \
  114. "\n#include added by oe-selftest.py\ninclude machine.inc\ninclude selftest.inc")
  115. if "#include added by oe-selftest.py" \
  116. not in ftools.read_file(os.path.join(builddir, "conf/bblayers.conf")):
  117. log.info("Adding: \"include bblayers.inc\" in bblayers.conf")
  118. ftools.append_file(os.path.join(builddir, "conf/bblayers.conf"), \
  119. "\n#include added by oe-selftest.py\ninclude bblayers.inc")
  120. def remove_include():
  121. builddir = os.environ.get("BUILDDIR")
  122. if builddir is None:
  123. return
  124. if "#include added by oe-selftest.py" \
  125. in ftools.read_file(os.path.join(builddir, "conf/local.conf")):
  126. log.info("Removing the include from local.conf")
  127. ftools.remove_from_file(os.path.join(builddir, "conf/local.conf"), \
  128. "\n#include added by oe-selftest.py\ninclude machine.inc\ninclude selftest.inc")
  129. if "#include added by oe-selftest.py" \
  130. in ftools.read_file(os.path.join(builddir, "conf/bblayers.conf")):
  131. log.info("Removing the include from bblayers.conf")
  132. ftools.remove_from_file(os.path.join(builddir, "conf/bblayers.conf"), \
  133. "\n#include added by oe-selftest.py\ninclude bblayers.inc")
  134. def remove_inc_files():
  135. try:
  136. os.remove(os.path.join(os.environ.get("BUILDDIR"), "conf/selftest.inc"))
  137. for root, _, files in os.walk(get_test_layer()):
  138. for f in files:
  139. if f == 'test_recipe.inc':
  140. os.remove(os.path.join(root, f))
  141. except (AttributeError, OSError,) as e: # AttributeError may happen if BUILDDIR is not set
  142. pass
  143. for incl_file in ['conf/bblayers.inc', 'conf/machine.inc']:
  144. try:
  145. os.remove(os.path.join(os.environ.get("BUILDDIR"), incl_file))
  146. except:
  147. pass
  148. def get_tests_modules(include_hidden=False):
  149. modules_list = list()
  150. for modules_path in oeqa.selftest.__path__:
  151. for (p, d, f) in os.walk(modules_path):
  152. files = sorted([f for f in os.listdir(p) if f.endswith('.py') and not (f.startswith('_') and not include_hidden) and not f.startswith('__') and f != 'base.py'])
  153. for f in files:
  154. submodules = p.split("selftest")[-1]
  155. module = ""
  156. if submodules:
  157. module = 'oeqa.selftest' + submodules.replace("/",".") + "." + f.split('.py')[0]
  158. else:
  159. module = 'oeqa.selftest.' + f.split('.py')[0]
  160. if module not in modules_list:
  161. modules_list.append(module)
  162. return modules_list
  163. def get_tests(exclusive_modules=[], include_hidden=False):
  164. test_modules = list()
  165. for x in exclusive_modules:
  166. test_modules.append('oeqa.selftest.' + x)
  167. if not test_modules:
  168. inc_hidden = include_hidden
  169. test_modules = get_tests_modules(inc_hidden)
  170. return test_modules
  171. class Tc:
  172. def __init__(self, tcname, tcclass, tcmodule, tcid=None, tctag=None):
  173. self.tcname = tcname
  174. self.tcclass = tcclass
  175. self.tcmodule = tcmodule
  176. self.tcid = tcid
  177. # A test case can have multiple tags (as tuples) otherwise str will suffice
  178. self.tctag = tctag
  179. self.fullpath = '.'.join(['oeqa', 'selftest', tcmodule, tcclass, tcname])
  180. def get_tests_from_module(tmod):
  181. tlist = []
  182. prefix = 'oeqa.selftest.'
  183. try:
  184. import importlib
  185. modlib = importlib.import_module(tmod)
  186. for mod in list(vars(modlib).values()):
  187. if isinstance(mod, type(oeSelfTest)) and issubclass(mod, oeSelfTest) and mod is not oeSelfTest:
  188. for test in dir(mod):
  189. if test.startswith('test_') and hasattr(vars(mod)[test], '__call__'):
  190. # Get test case id and feature tag
  191. # NOTE: if testcase decorator or feature tag not set will throw error
  192. try:
  193. tid = vars(mod)[test].test_case
  194. except:
  195. print('DEBUG: tc id missing for ' + str(test))
  196. tid = None
  197. try:
  198. ttag = vars(mod)[test].tag__feature
  199. except:
  200. # print('DEBUG: feature tag missing for ' + str(test))
  201. ttag = None
  202. # NOTE: for some reason lstrip() doesn't work for mod.__module__
  203. tlist.append(Tc(test, mod.__name__, mod.__module__.replace(prefix, ''), tid, ttag))
  204. except:
  205. pass
  206. return tlist
  207. def get_all_tests():
  208. # Get all the test modules (except the hidden ones)
  209. testlist = []
  210. tests_modules = get_tests_modules()
  211. # Get all the tests from modules
  212. for tmod in sorted(tests_modules):
  213. testlist += get_tests_from_module(tmod)
  214. return testlist
  215. def get_testsuite_by(criteria, keyword):
  216. # Get a testsuite based on 'keyword'
  217. # criteria: name, class, module, id, tag
  218. # keyword: a list of tests, classes, modules, ids, tags
  219. ts = []
  220. all_tests = get_all_tests()
  221. def get_matches(values):
  222. # Get an item and return the ones that match with keyword(s)
  223. # values: the list of items (names, modules, classes...)
  224. result = []
  225. remaining = values[:]
  226. for key in keyword:
  227. found = False
  228. if key in remaining:
  229. # Regular matching of exact item
  230. result.append(key)
  231. remaining.remove(key)
  232. found = True
  233. else:
  234. # Wildcard matching
  235. pattern = re.compile(fnmatch.translate(r"%s" % key))
  236. added = [x for x in remaining if pattern.match(x)]
  237. if added:
  238. result.extend(added)
  239. remaining = [x for x in remaining if x not in added]
  240. found = True
  241. if not found:
  242. log.error("Failed to find test: %s" % key)
  243. return result
  244. if criteria == 'name':
  245. names = get_matches([ tc.tcname for tc in all_tests ])
  246. ts = [ tc for tc in all_tests if tc.tcname in names ]
  247. elif criteria == 'class':
  248. classes = get_matches([ tc.tcclass for tc in all_tests ])
  249. ts = [ tc for tc in all_tests if tc.tcclass in classes ]
  250. elif criteria == 'module':
  251. modules = get_matches([ tc.tcmodule for tc in all_tests ])
  252. ts = [ tc for tc in all_tests if tc.tcmodule in modules ]
  253. elif criteria == 'id':
  254. ids = get_matches([ str(tc.tcid) for tc in all_tests ])
  255. ts = [ tc for tc in all_tests if str(tc.tcid) in ids ]
  256. elif criteria == 'tag':
  257. values = set()
  258. for tc in all_tests:
  259. # tc can have multiple tags (as tuple) otherwise str will suffice
  260. if isinstance(tc.tctag, tuple):
  261. values |= { str(tag) for tag in tc.tctag }
  262. else:
  263. values.add(str(tc.tctag))
  264. tags = get_matches(list(values))
  265. for tc in all_tests:
  266. for tag in tags:
  267. if isinstance(tc.tctag, tuple) and tag in tc.tctag:
  268. ts.append(tc)
  269. elif tag == tc.tctag:
  270. ts.append(tc)
  271. # Remove duplicates from the list
  272. ts = list(set(ts))
  273. return ts
  274. def list_testsuite_by(criteria, keyword):
  275. # Get a testsuite based on 'keyword'
  276. # criteria: name, class, module, id, tag
  277. # keyword: a list of tests, classes, modules, ids, tags
  278. ts = sorted([ (tc.tcid, tc.tctag, tc.tcname, tc.tcclass, tc.tcmodule) for tc in get_testsuite_by(criteria, keyword) ])
  279. print('%-4s\t%-20s\t%-60s\t%-25s\t%-20s' % ('id', 'tag', 'name', 'class', 'module'))
  280. print('_' * 150)
  281. for t in ts:
  282. if isinstance(t[1], (tuple, list)):
  283. print('%-4s\t%-20s\t%-60s\t%-25s\t%-20s' % (t[0], ', '.join(t[1]), t[2], t[3], t[4]))
  284. else:
  285. print('%-4s\t%-20s\t%-60s\t%-25s\t%-20s' % t)
  286. print('_' * 150)
  287. print('Filtering by:\t %s' % criteria)
  288. print('Looking for:\t %s' % ', '.join(str(x) for x in keyword))
  289. print('Total found:\t %s' % len(ts))
  290. def list_tests():
  291. # List all available oe-selftest tests
  292. ts = get_all_tests()
  293. print('%-4s\t%-10s\t%-50s' % ('id', 'tag', 'test'))
  294. print('_' * 80)
  295. for t in ts:
  296. if isinstance(t.tctag, (tuple, list)):
  297. print('%-4s\t%-10s\t%-50s' % (t.tcid, ', '.join(t.tctag), '.'.join([t.tcmodule, t.tcclass, t.tcname])))
  298. else:
  299. print('%-4s\t%-10s\t%-50s' % (t.tcid, t.tctag, '.'.join([t.tcmodule, t.tcclass, t.tcname])))
  300. print('_' * 80)
  301. print('Total found:\t %s' % len(ts))
  302. def list_tags():
  303. # Get all tags set to test cases
  304. # This is useful when setting tags to test cases
  305. # The list of tags should be kept as minimal as possible
  306. tags = set()
  307. all_tests = get_all_tests()
  308. for tc in all_tests:
  309. if isinstance(tc.tctag, (tuple, list)):
  310. tags.update(set(tc.tctag))
  311. else:
  312. tags.add(tc.tctag)
  313. print('Tags:\t%s' % ', '.join(str(x) for x in tags))
  314. def coverage_setup(coverage_source, coverage_include, coverage_omit):
  315. """ Set up the coverage measurement for the testcases to be run """
  316. import datetime
  317. import subprocess
  318. builddir = os.environ.get("BUILDDIR")
  319. pokydir = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
  320. curcommit= subprocess.check_output(["git", "--git-dir", os.path.join(pokydir, ".git"), "rev-parse", "HEAD"]).decode('utf-8')
  321. coveragerc = "%s/.coveragerc" % builddir
  322. data_file = "%s/.coverage." % builddir
  323. data_file += datetime.datetime.now().strftime('%Y%m%dT%H%M%S')
  324. if os.path.isfile(data_file):
  325. os.remove(data_file)
  326. with open(coveragerc, 'w') as cps:
  327. cps.write("# Generated with command '%s'\n" % " ".join(sys.argv))
  328. cps.write("# HEAD commit %s\n" % curcommit.strip())
  329. cps.write("[run]\n")
  330. cps.write("data_file = %s\n" % data_file)
  331. cps.write("branch = True\n")
  332. # Measure just BBLAYERS, scripts and bitbake folders
  333. cps.write("source = \n")
  334. if coverage_source:
  335. for directory in coverage_source:
  336. if not os.path.isdir(directory):
  337. log.warn("Directory %s is not valid.", directory)
  338. cps.write(" %s\n" % directory)
  339. else:
  340. for layer in get_bb_var('BBLAYERS').split():
  341. cps.write(" %s\n" % layer)
  342. cps.write(" %s\n" % os.path.dirname(os.path.realpath(__file__)))
  343. cps.write(" %s\n" % os.path.join(os.path.dirname(os.path.dirname(os.path.realpath(__file__))),'bitbake'))
  344. if coverage_include:
  345. cps.write("include = \n")
  346. for pattern in coverage_include:
  347. cps.write(" %s\n" % pattern)
  348. if coverage_omit:
  349. cps.write("omit = \n")
  350. for pattern in coverage_omit:
  351. cps.write(" %s\n" % pattern)
  352. return coveragerc
  353. def coverage_report():
  354. """ Loads the coverage data gathered and reports it back """
  355. try:
  356. # Coverage4 uses coverage.Coverage
  357. from coverage import Coverage
  358. except:
  359. # Coverage under version 4 uses coverage.coverage
  360. from coverage import coverage as Coverage
  361. import io as StringIO
  362. from coverage.misc import CoverageException
  363. cov_output = StringIO.StringIO()
  364. # Creating the coverage data with the setting from the configuration file
  365. cov = Coverage(config_file = os.environ.get('COVERAGE_PROCESS_START'))
  366. try:
  367. # Load data from the data file specified in the configuration
  368. cov.load()
  369. # Store report data in a StringIO variable
  370. cov.report(file = cov_output, show_missing=False)
  371. log.info("\n%s" % cov_output.getvalue())
  372. except CoverageException as e:
  373. # Show problems with the reporting. Since Coverage4 not finding any data to report raises an exception
  374. log.warn("%s" % str(e))
  375. finally:
  376. cov_output.close()
  377. def main():
  378. parser = get_args_parser()
  379. args = parser.parse_args()
  380. # Add <layer>/lib to sys.path, so layers can add selftests
  381. log.info("Running bitbake -e to get BBPATH")
  382. bbpath = get_bb_var('BBPATH').split(':')
  383. layer_libdirs = [p for p in (os.path.join(l, 'lib') for l in bbpath) if os.path.exists(p)]
  384. sys.path.extend(layer_libdirs)
  385. imp.reload(oeqa.selftest)
  386. if args.run_tests_by and len(args.run_tests_by) >= 2:
  387. valid_options = ['name', 'class', 'module', 'id', 'tag']
  388. if args.run_tests_by[0] not in valid_options:
  389. print('--run-tests-by %s not a valid option. Choose one of <name|class|module|id|tag>.' % args.run_tests_by[0])
  390. return 1
  391. else:
  392. criteria = args.run_tests_by[0]
  393. keyword = args.run_tests_by[1:]
  394. ts = sorted([ tc.fullpath for tc in get_testsuite_by(criteria, keyword) ])
  395. if not ts:
  396. return 1
  397. if args.list_tests_by and len(args.list_tests_by) >= 2:
  398. valid_options = ['name', 'class', 'module', 'id', 'tag']
  399. if args.list_tests_by[0] not in valid_options:
  400. print('--list-tests-by %s not a valid option. Choose one of <name|class|module|id|tag>.' % args.list_tests_by[0])
  401. return 1
  402. else:
  403. criteria = args.list_tests_by[0]
  404. keyword = args.list_tests_by[1:]
  405. list_testsuite_by(criteria, keyword)
  406. if args.list_tests:
  407. list_tests()
  408. if args.list_tags:
  409. list_tags()
  410. if args.list_allclasses:
  411. args.list_modules = True
  412. if args.list_modules:
  413. log.info('Listing all available test modules:')
  414. testslist = get_tests(include_hidden=True)
  415. for test in testslist:
  416. module = test.split('oeqa.selftest.')[-1]
  417. info = ''
  418. if module.startswith('_'):
  419. info = ' (hidden)'
  420. print(module + info)
  421. if args.list_allclasses:
  422. try:
  423. import importlib
  424. modlib = importlib.import_module(test)
  425. for v in vars(modlib):
  426. t = vars(modlib)[v]
  427. if isinstance(t, type(oeSelfTest)) and issubclass(t, oeSelfTest) and t!=oeSelfTest:
  428. print(" --", v)
  429. for method in dir(t):
  430. if method.startswith("test_") and isinstance(vars(t)[method], collections.Callable):
  431. print(" -- --", method)
  432. except (AttributeError, ImportError) as e:
  433. print(e)
  434. pass
  435. if args.run_tests or args.run_all_tests or args.run_tests_by:
  436. if not preflight_check():
  437. return 1
  438. if args.run_tests_by:
  439. testslist = ts
  440. else:
  441. testslist = get_tests(exclusive_modules=(args.run_tests or []), include_hidden=False)
  442. suite = unittest.TestSuite()
  443. loader = unittest.TestLoader()
  444. loader.sortTestMethodsUsing = None
  445. runner = TestRunner(verbosity=2,
  446. resultclass=buildResultClass(args))
  447. # we need to do this here, otherwise just loading the tests
  448. # will take 2 minutes (bitbake -e calls)
  449. oeSelfTest.testlayer_path = get_test_layer()
  450. for test in testslist:
  451. log.info("Loading tests from: %s" % test)
  452. try:
  453. suite.addTests(loader.loadTestsFromName(test))
  454. except AttributeError as e:
  455. log.error("Failed to import %s" % test)
  456. log.error(e)
  457. return 1
  458. add_include()
  459. if args.machine:
  460. # Custom machine sets only weak default values (??=) for MACHINE in machine.inc
  461. # This let test cases that require a specific MACHINE to be able to override it, using (?= or =)
  462. log.info('Custom machine mode enabled. MACHINE set to %s' % args.machine)
  463. if args.machine == 'random':
  464. os.environ['CUSTOMMACHINE'] = 'random'
  465. result = runner.run(suite)
  466. else: # all
  467. machines = get_available_machines()
  468. for m in machines:
  469. log.info('Run tests with custom MACHINE set to: %s' % m)
  470. os.environ['CUSTOMMACHINE'] = m
  471. result = runner.run(suite)
  472. else:
  473. result = runner.run(suite)
  474. log.info("Finished")
  475. if result.wasSuccessful():
  476. return 0
  477. else:
  478. return 1
  479. def buildResultClass(args):
  480. """Build a Result Class to use in the testcase execution"""
  481. import site
  482. class StampedResult(TestResult):
  483. """
  484. Custom TestResult that prints the time when a test starts. As oe-selftest
  485. can take a long time (ie a few hours) to run, timestamps help us understand
  486. what tests are taking a long time to execute.
  487. If coverage is required, this class executes the coverage setup and reporting.
  488. """
  489. def startTest(self, test):
  490. import time
  491. self.stream.write(time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()) + " - ")
  492. super(StampedResult, self).startTest(test)
  493. def startTestRun(self):
  494. """ Setup coverage before running any testcase """
  495. # variable holding the coverage configuration file allowing subprocess to be measured
  496. self.coveragepth = None
  497. # indicates the system if coverage is currently installed
  498. self.coverage_installed = True
  499. if args.coverage or args.coverage_source or args.coverage_include or args.coverage_omit:
  500. try:
  501. # check if user can do coverage
  502. import coverage
  503. except:
  504. log.warn("python coverage is not installed. More info on https://pypi.python.org/pypi/coverage")
  505. self.coverage_installed = False
  506. if self.coverage_installed:
  507. log.info("Coverage is enabled")
  508. major_version = int(coverage.version.__version__[0])
  509. if major_version < 4:
  510. log.error("python coverage %s installed. Require version 4 or greater." % coverage.version.__version__)
  511. self.stop()
  512. # In case the user has not set the variable COVERAGE_PROCESS_START,
  513. # create a default one and export it. The COVERAGE_PROCESS_START
  514. # value indicates where the coverage configuration file resides
  515. # More info on https://pypi.python.org/pypi/coverage
  516. if not os.environ.get('COVERAGE_PROCESS_START'):
  517. os.environ['COVERAGE_PROCESS_START'] = coverage_setup(args.coverage_source, args.coverage_include, args.coverage_omit)
  518. # Use default site.USER_SITE and write corresponding config file
  519. site.ENABLE_USER_SITE = True
  520. if not os.path.exists(site.USER_SITE):
  521. os.makedirs(site.USER_SITE)
  522. self.coveragepth = os.path.join(site.USER_SITE, "coverage.pth")
  523. with open(self.coveragepth, 'w') as cps:
  524. cps.write('import sys,site; sys.path.extend(site.getsitepackages()); import coverage; coverage.process_startup();')
  525. def stopTestRun(self):
  526. """ Report coverage data after the testcases are run """
  527. if args.coverage or args.coverage_source or args.coverage_include or args.coverage_omit:
  528. if self.coverage_installed:
  529. with open(os.environ['COVERAGE_PROCESS_START']) as ccf:
  530. log.info("Coverage configuration file (%s)" % os.environ.get('COVERAGE_PROCESS_START'))
  531. log.info("===========================")
  532. log.info("\n%s" % "".join(ccf.readlines()))
  533. log.info("Coverage Report")
  534. log.info("===============")
  535. try:
  536. coverage_report()
  537. finally:
  538. # remove the pth file
  539. try:
  540. os.remove(self.coveragepth)
  541. except OSError:
  542. log.warn("Expected temporal file from coverage is missing, ignoring removal.")
  543. return StampedResult
  544. class TestRunner(_TestRunner):
  545. """Test runner class aware of exporting tests."""
  546. def __init__(self, *args, **kwargs):
  547. try:
  548. exportdir = os.path.join(os.getcwd(), log_prefix)
  549. kwargsx = dict(**kwargs)
  550. # argument specific to XMLTestRunner, if adding a new runner then
  551. # also add logic to use other runner's args.
  552. kwargsx['output'] = exportdir
  553. kwargsx['descriptions'] = False
  554. # done for the case where telling the runner where to export
  555. super(TestRunner, self).__init__(*args, **kwargsx)
  556. except TypeError:
  557. log.info("test runner init'ed like unittest")
  558. super(TestRunner, self).__init__(*args, **kwargs)
  559. if __name__ == "__main__":
  560. try:
  561. ret = main()
  562. except Exception:
  563. ret = 1
  564. import traceback
  565. traceback.print_exc()
  566. finally:
  567. remove_include()
  568. remove_inc_files()
  569. sys.exit(ret)