loader.py 14 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369
  1. #
  2. # Copyright (C) 2016 Intel Corporation
  3. #
  4. # SPDX-License-Identifier: MIT
  5. #
  6. import os
  7. import re
  8. import sys
  9. import unittest
  10. import inspect
  11. from oeqa.core.utils.path import findFile
  12. from oeqa.core.utils.test import getSuiteModules, getCaseID
  13. from oeqa.core.exception import OEQATestNotFound
  14. from oeqa.core.case import OETestCase
  15. from oeqa.core.decorator import decoratorClasses, OETestDecorator, \
  16. OETestFilter, OETestDiscover
  17. # When loading tests, the unittest framework stores any exceptions and
  18. # displays them only when the run method is called.
  19. #
  20. # For our purposes, it is better to raise the exceptions in the loading
  21. # step rather than waiting to run the test suite.
  22. #
  23. # Generate the function definition because this differ across python versions
  24. # Python >= 3.4.4 uses tree parameters instead four but for example Python 3.5.3
  25. # ueses four parameters so isn't incremental.
  26. _failed_test_args = inspect.getfullargspec(unittest.loader._make_failed_test).args
  27. exec("""def _make_failed_test(%s): raise exception""" % ', '.join(_failed_test_args))
  28. unittest.loader._make_failed_test = _make_failed_test
  29. def _find_duplicated_modules(suite, directory):
  30. for module in getSuiteModules(suite):
  31. path = findFile('%s.py' % module, directory)
  32. if path:
  33. raise ImportError("Duplicated %s module found in %s" % (module, path))
  34. def _built_modules_dict(modules):
  35. modules_dict = {}
  36. if modules == None:
  37. return modules_dict
  38. for module in modules:
  39. # Assumption: package and module names do not contain upper case
  40. # characters, whereas class names do
  41. m = re.match(r'^(\w+)(?:\.(\w[^.]*)(?:\.([^.]+))?)?$', module, flags=re.ASCII)
  42. if not m:
  43. continue
  44. module_name, class_name, test_name = m.groups()
  45. if module_name and module_name not in modules_dict:
  46. modules_dict[module_name] = {}
  47. if class_name and class_name not in modules_dict[module_name]:
  48. modules_dict[module_name][class_name] = []
  49. if test_name and test_name not in modules_dict[module_name][class_name]:
  50. modules_dict[module_name][class_name].append(test_name)
  51. return modules_dict
  52. class OETestLoader(unittest.TestLoader):
  53. caseClass = OETestCase
  54. kwargs_names = ['testMethodPrefix', 'sortTestMethodUsing', 'suiteClass',
  55. '_top_level_dir']
  56. def __init__(self, tc, module_paths, modules, tests, modules_required,
  57. filters, *args, **kwargs):
  58. self.tc = tc
  59. self.modules = _built_modules_dict(modules)
  60. self.tests = tests
  61. self.modules_required = modules_required
  62. self.filters = filters
  63. self.decorator_filters = [d for d in decoratorClasses if \
  64. issubclass(d, OETestFilter)]
  65. self._validateFilters(self.filters, self.decorator_filters)
  66. self.used_filters = [d for d in self.decorator_filters
  67. for f in self.filters
  68. if f in d.attrs]
  69. if isinstance(module_paths, str):
  70. module_paths = [module_paths]
  71. elif not isinstance(module_paths, list):
  72. raise TypeError('module_paths must be a str or a list of str')
  73. self.module_paths = module_paths
  74. for kwname in self.kwargs_names:
  75. if kwname in kwargs:
  76. setattr(self, kwname, kwargs[kwname])
  77. self._patchCaseClass(self.caseClass)
  78. super(OETestLoader, self).__init__()
  79. def _patchCaseClass(self, testCaseClass):
  80. # Adds custom attributes to the OETestCase class
  81. setattr(testCaseClass, 'tc', self.tc)
  82. setattr(testCaseClass, 'td', self.tc.td)
  83. setattr(testCaseClass, 'logger', self.tc.logger)
  84. def _validateFilters(self, filters, decorator_filters):
  85. # Validate if filter isn't empty
  86. for key,value in filters.items():
  87. if not value:
  88. raise TypeError("Filter %s specified is empty" % key)
  89. # Validate unique attributes
  90. attr_filters = [attr for clss in decorator_filters \
  91. for attr in clss.attrs]
  92. dup_attr = [attr for attr in attr_filters
  93. if attr_filters.count(attr) > 1]
  94. if dup_attr:
  95. raise TypeError('Detected duplicated attribute(s) %s in filter'
  96. ' decorators' % ' ,'.join(dup_attr))
  97. # Validate if filter is supported
  98. for f in filters:
  99. if f not in attr_filters:
  100. classes = ', '.join([d.__name__ for d in decorator_filters])
  101. raise TypeError('Found "%s" filter but not declared in any of '
  102. '%s decorators' % (f, classes))
  103. def _registerTestCase(self, case):
  104. case_id = case.id()
  105. self.tc._registry['cases'][case_id] = case
  106. def _handleTestCaseDecorators(self, case):
  107. def _handle(obj):
  108. if isinstance(obj, OETestDecorator):
  109. if not obj.__class__ in decoratorClasses:
  110. raise Exception("Decorator %s isn't registered" \
  111. " in decoratorClasses." % obj.__name__)
  112. obj.bind(self.tc._registry, case)
  113. def _walk_closure(obj):
  114. if hasattr(obj, '__closure__') and obj.__closure__:
  115. for f in obj.__closure__:
  116. obj = f.cell_contents
  117. _handle(obj)
  118. _walk_closure(obj)
  119. method = getattr(case, case._testMethodName, None)
  120. _walk_closure(method)
  121. def _filterTest(self, case):
  122. """
  123. Returns True if test case must be filtered, False otherwise.
  124. """
  125. # XXX; If the module has more than one namespace only use
  126. # the first to support run the whole module specifying the
  127. # <module_name>.[test_class].[test_name]
  128. module_name_small = case.__module__.split('.')[0]
  129. module_name = case.__module__
  130. class_name = case.__class__.__name__
  131. test_name = case._testMethodName
  132. # 'auto' is a reserved key word to run test cases automatically
  133. # warn users if their test case belong to a module named 'auto'
  134. if module_name_small == "auto":
  135. bb.warn("'auto' is a reserved key word for TEST_SUITES. "
  136. "But test case '%s' is detected to belong to auto module. "
  137. "Please condier using a new name for your module." % str(case))
  138. # check if case belongs to any specified module
  139. # if 'auto' is specified, such check is skipped
  140. if self.modules and not 'auto' in self.modules:
  141. module = None
  142. try:
  143. module = self.modules[module_name_small]
  144. except KeyError:
  145. try:
  146. module = self.modules[module_name]
  147. except KeyError:
  148. return True
  149. if module:
  150. if not class_name in module:
  151. return True
  152. if module[class_name]:
  153. if test_name not in module[class_name]:
  154. return True
  155. # Decorator filters
  156. if self.filters and isinstance(case, OETestCase):
  157. filters = self.filters.copy()
  158. case_decorators = [cd for cd in case.decorators
  159. if cd.__class__ in self.used_filters]
  160. # Iterate over case decorators to check if needs to be filtered.
  161. for cd in case_decorators:
  162. if cd.filtrate(filters):
  163. return True
  164. # Case is missing one or more decorators for all the filters
  165. # being used, so filter test case.
  166. if filters:
  167. return True
  168. return False
  169. def _getTestCase(self, testCaseClass, tcName):
  170. if not hasattr(testCaseClass, '__oeqa_loader') and \
  171. issubclass(testCaseClass, OETestCase):
  172. # In order to support data_vars validation
  173. # monkey patch the default setUp/tearDown{Class} to use
  174. # the ones provided by OETestCase
  175. setattr(testCaseClass, 'setUpClassMethod',
  176. getattr(testCaseClass, 'setUpClass'))
  177. setattr(testCaseClass, 'tearDownClassMethod',
  178. getattr(testCaseClass, 'tearDownClass'))
  179. setattr(testCaseClass, 'setUpClass',
  180. testCaseClass._oeSetUpClass)
  181. setattr(testCaseClass, 'tearDownClass',
  182. testCaseClass._oeTearDownClass)
  183. # In order to support decorators initialization
  184. # monkey patch the default setUp/tearDown to use
  185. # a setUpDecorators/tearDownDecorators that methods
  186. # will call setUp/tearDown original methods.
  187. setattr(testCaseClass, 'setUpMethod',
  188. getattr(testCaseClass, 'setUp'))
  189. setattr(testCaseClass, 'tearDownMethod',
  190. getattr(testCaseClass, 'tearDown'))
  191. setattr(testCaseClass, 'setUp', testCaseClass._oeSetUp)
  192. setattr(testCaseClass, 'tearDown', testCaseClass._oeTearDown)
  193. setattr(testCaseClass, '__oeqa_loader', True)
  194. case = testCaseClass(tcName)
  195. if isinstance(case, OETestCase):
  196. setattr(case, 'decorators', [])
  197. return case
  198. def loadTestsFromTestCase(self, testCaseClass):
  199. """
  200. Returns a suite of all tests cases contained in testCaseClass.
  201. """
  202. if issubclass(testCaseClass, unittest.suite.TestSuite):
  203. raise TypeError("Test cases should not be derived from TestSuite." \
  204. " Maybe you meant to derive %s from TestCase?" \
  205. % testCaseClass.__name__)
  206. if not issubclass(testCaseClass, unittest.case.TestCase):
  207. raise TypeError("Test %s is not derived from %s" % \
  208. (testCaseClass.__name__, unittest.case.TestCase.__name__))
  209. testCaseNames = self.getTestCaseNames(testCaseClass)
  210. if not testCaseNames and hasattr(testCaseClass, 'runTest'):
  211. testCaseNames = ['runTest']
  212. suite = []
  213. for tcName in testCaseNames:
  214. case = self._getTestCase(testCaseClass, tcName)
  215. # Filer by case id
  216. if not (self.tests and not 'auto' in self.tests
  217. and not getCaseID(case) in self.tests):
  218. self._handleTestCaseDecorators(case)
  219. # Filter by decorators
  220. if not self._filterTest(case):
  221. self._registerTestCase(case)
  222. suite.append(case)
  223. return self.suiteClass(suite)
  224. def _required_modules_validation(self):
  225. """
  226. Search in Test context registry if a required
  227. test is found, raise an exception when not found.
  228. """
  229. for module in self.modules_required:
  230. found = False
  231. # The module name is splitted to only compare the
  232. # first part of a test case id.
  233. comp_len = len(module.split('.'))
  234. for case in self.tc._registry['cases']:
  235. case_comp = '.'.join(case.split('.')[0:comp_len])
  236. if module == case_comp:
  237. found = True
  238. break
  239. if not found:
  240. raise OEQATestNotFound("Not found %s in loaded test cases" % \
  241. module)
  242. def discover(self):
  243. big_suite = self.suiteClass()
  244. for path in self.module_paths:
  245. _find_duplicated_modules(big_suite, path)
  246. suite = super(OETestLoader, self).discover(path,
  247. pattern='*.py', top_level_dir=path)
  248. big_suite.addTests(suite)
  249. cases = None
  250. discover_classes = [clss for clss in decoratorClasses
  251. if issubclass(clss, OETestDiscover)]
  252. for clss in discover_classes:
  253. cases = clss.discover(self.tc._registry)
  254. if self.modules_required:
  255. self._required_modules_validation()
  256. return self.suiteClass(cases) if cases else big_suite
  257. def _filterModule(self, module):
  258. if module.__name__ in sys.builtin_module_names:
  259. msg = 'Tried to import %s test module but is a built-in'
  260. raise ImportError(msg % module.__name__)
  261. # XXX; If the module has more than one namespace only use
  262. # the first to support run the whole module specifying the
  263. # <module_name>.[test_class].[test_name]
  264. module_name_small = module.__name__.split('.')[0]
  265. module_name = module.__name__
  266. # Normal test modules are loaded if no modules were specified,
  267. # if module is in the specified module list or if 'auto' is in
  268. # module list.
  269. # Underscore modules are loaded only if specified in module list.
  270. load_module = True if not module_name.startswith('_') \
  271. and (not self.modules \
  272. or module_name in self.modules \
  273. or module_name_small in self.modules \
  274. or 'auto' in self.modules) \
  275. else False
  276. load_underscore = True if module_name.startswith('_') \
  277. and (module_name in self.modules or \
  278. module_name_small in self.modules) \
  279. else False
  280. return (load_module, load_underscore)
  281. # XXX After Python 3.5, remove backward compatibility hacks for
  282. # use_load_tests deprecation via *args and **kws. See issue 16662.
  283. if sys.version_info >= (3,5):
  284. def loadTestsFromModule(self, module, *args, pattern=None, **kws):
  285. """
  286. Returns a suite of all tests cases contained in module.
  287. """
  288. load_module, load_underscore = self._filterModule(module)
  289. if load_module or load_underscore:
  290. return super(OETestLoader, self).loadTestsFromModule(
  291. module, *args, pattern=pattern, **kws)
  292. else:
  293. return self.suiteClass()
  294. else:
  295. def loadTestsFromModule(self, module, use_load_tests=True):
  296. """
  297. Returns a suite of all tests cases contained in module.
  298. """
  299. load_module, load_underscore = self._filterModule(module)
  300. if load_module or load_underscore:
  301. return super(OETestLoader, self).loadTestsFromModule(
  302. module, use_load_tests)
  303. else:
  304. return self.suiteClass()