oetest.py 21 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618
  1. #
  2. # Copyright (C) 2013 Intel Corporation
  3. #
  4. # SPDX-License-Identifier: MIT
  5. #
  6. # Main unittest module used by testimage.bbclass
  7. # This provides the oeRuntimeTest base class which is inherited by all tests in meta/lib/oeqa/runtime.
  8. # It also has some helper functions and it's responsible for actually starting the tests
  9. import os, re, sys
  10. import unittest
  11. import inspect
  12. import subprocess
  13. import signal
  14. import shutil
  15. import functools
  16. try:
  17. import bb
  18. except ImportError:
  19. pass
  20. import logging
  21. import oeqa.runtime
  22. # Exported test doesn't require sdkext
  23. try:
  24. import oeqa.sdkext
  25. except ImportError:
  26. pass
  27. from oeqa.utils.decorators import LogResults, gettag, getResults
  28. logger = logging.getLogger("BitBake")
  29. def getVar(obj):
  30. #extend form dict, if a variable didn't exists, need find it in testcase
  31. class VarDict(dict):
  32. def __getitem__(self, key):
  33. return gettag(obj, key)
  34. return VarDict()
  35. def checkTags(tc, tagexp):
  36. return eval(tagexp, None, getVar(tc))
  37. def filterByTagExp(testsuite, tagexp):
  38. if not tagexp:
  39. return testsuite
  40. caseList = []
  41. for each in testsuite:
  42. if not isinstance(each, unittest.BaseTestSuite):
  43. if checkTags(each, tagexp):
  44. caseList.append(each)
  45. else:
  46. caseList.append(filterByTagExp(each, tagexp))
  47. return testsuite.__class__(caseList)
  48. @LogResults
  49. class oeTest(unittest.TestCase):
  50. pscmd = "ps"
  51. longMessage = True
  52. @classmethod
  53. def hasPackage(self, pkg):
  54. """
  55. True if the full package name exists in the manifest, False otherwise.
  56. """
  57. return pkg in oeTest.tc.pkgmanifest
  58. @classmethod
  59. def hasPackageMatch(self, match):
  60. """
  61. True if match exists in the manifest as a regular expression substring,
  62. False otherwise.
  63. """
  64. for s in oeTest.tc.pkgmanifest:
  65. if re.match(match, s):
  66. return True
  67. return False
  68. @classmethod
  69. def hasFeature(self,feature):
  70. if feature in oeTest.tc.imagefeatures or \
  71. feature in oeTest.tc.distrofeatures:
  72. return True
  73. else:
  74. return False
  75. class oeRuntimeTest(oeTest):
  76. def __init__(self, methodName='runTest'):
  77. self.target = oeRuntimeTest.tc.target
  78. super(oeRuntimeTest, self).__init__(methodName)
  79. def setUp(self):
  80. # Install packages in the DUT
  81. self.tc.install_uninstall_packages(self.id())
  82. # Check if test needs to run
  83. if self.tc.sigterm:
  84. self.fail("Got SIGTERM")
  85. elif (type(self.target).__name__ == "QemuTarget"):
  86. self.assertTrue(self.target.check(), msg = "Qemu not running?")
  87. self.setUpLocal()
  88. # a setup method before tests but after the class instantiation
  89. def setUpLocal(self):
  90. pass
  91. def tearDown(self):
  92. # Uninstall packages in the DUT
  93. self.tc.install_uninstall_packages(self.id(), False)
  94. res = getResults()
  95. # If a test fails or there is an exception dump
  96. # for QemuTarget only
  97. if (type(self.target).__name__ == "QemuTarget" and
  98. (self.id() in res.getErrorList() or
  99. self.id() in res.getFailList())):
  100. self.tc.host_dumper.create_dir(self._testMethodName)
  101. self.tc.host_dumper.dump_host()
  102. self.target.target_dumper.dump_target(
  103. self.tc.host_dumper.dump_dir)
  104. print ("%s dump data stored in %s" % (self._testMethodName,
  105. self.tc.host_dumper.dump_dir))
  106. self.tearDownLocal()
  107. # Method to be run after tearDown and implemented by child classes
  108. def tearDownLocal(self):
  109. pass
  110. def getmodule(pos=2):
  111. # stack returns a list of tuples containg frame information
  112. # First element of the list the is current frame, caller is 1
  113. frameinfo = inspect.stack()[pos]
  114. modname = inspect.getmodulename(frameinfo[1])
  115. #modname = inspect.getmodule(frameinfo[0]).__name__
  116. return modname
  117. def skipModule(reason, pos=2):
  118. modname = getmodule(pos)
  119. if modname not in oeTest.tc.testsrequired:
  120. raise unittest.SkipTest("%s: %s" % (modname, reason))
  121. else:
  122. raise Exception("\nTest %s wants to be skipped.\nReason is: %s" \
  123. "\nTest was required in TEST_SUITES, so either the condition for skipping is wrong" \
  124. "\nor the image really doesn't have the required feature/package when it should." % (modname, reason))
  125. def skipModuleIf(cond, reason):
  126. if cond:
  127. skipModule(reason, 3)
  128. def skipModuleUnless(cond, reason):
  129. if not cond:
  130. skipModule(reason, 3)
  131. _buffer_logger = ""
  132. def custom_verbose(msg, *args, **kwargs):
  133. global _buffer_logger
  134. if msg[-1] != "\n":
  135. _buffer_logger += msg
  136. else:
  137. _buffer_logger += msg
  138. try:
  139. bb.plain(_buffer_logger.rstrip("\n"), *args, **kwargs)
  140. except NameError:
  141. logger.info(_buffer_logger.rstrip("\n"), *args, **kwargs)
  142. _buffer_logger = ""
  143. class TestContext(object):
  144. def __init__(self, d, exported=False):
  145. self.d = d
  146. self.testsuites = self._get_test_suites()
  147. if exported:
  148. path = [os.path.dirname(os.path.abspath(__file__))]
  149. extrapath = ""
  150. else:
  151. path = d.getVar("BBPATH").split(':')
  152. extrapath = "lib/oeqa"
  153. self.testslist = self._get_tests_list(path, extrapath)
  154. self.testsrequired = self._get_test_suites_required()
  155. self.filesdir = os.path.join(os.path.dirname(os.path.abspath(__file__)), "runtime/files")
  156. self.corefilesdir = os.path.join(os.path.dirname(os.path.abspath(__file__)), "files")
  157. self.imagefeatures = d.getVar("IMAGE_FEATURES").split()
  158. self.distrofeatures = d.getVar("DISTRO_FEATURES").split()
  159. # get testcase list from specified file
  160. # if path is a relative path, then relative to build/conf/
  161. def _read_testlist(self, fpath, builddir):
  162. if not os.path.isabs(fpath):
  163. fpath = os.path.join(builddir, "conf", fpath)
  164. if not os.path.exists(fpath):
  165. bb.fatal("No such manifest file: ", fpath)
  166. tcs = []
  167. for line in open(fpath).readlines():
  168. line = line.strip()
  169. if line and not line.startswith("#"):
  170. tcs.append(line)
  171. return " ".join(tcs)
  172. # return test list by type also filter if TEST_SUITES is specified
  173. def _get_tests_list(self, bbpath, extrapath):
  174. testslist = []
  175. type = self._get_test_namespace()
  176. # This relies on lib/ under each directory in BBPATH being added to sys.path
  177. # (as done by default in base.bbclass)
  178. for testname in self.testsuites:
  179. if testname != "auto":
  180. if testname.startswith("oeqa."):
  181. testslist.append(testname)
  182. continue
  183. found = False
  184. for p in bbpath:
  185. if os.path.exists(os.path.join(p, extrapath, type, testname + ".py")):
  186. testslist.append("oeqa." + type + "." + testname)
  187. found = True
  188. break
  189. elif os.path.exists(os.path.join(p, extrapath, type, testname.split(".")[0] + ".py")):
  190. testslist.append("oeqa." + type + "." + testname)
  191. found = True
  192. break
  193. if not found:
  194. bb.fatal('Test %s specified in TEST_SUITES could not be found in lib/oeqa/runtime under BBPATH' % testname)
  195. if "auto" in self.testsuites:
  196. def add_auto_list(path):
  197. files = sorted([f for f in os.listdir(path) if f.endswith('.py') and not f.startswith('_')])
  198. for f in files:
  199. module = 'oeqa.' + type + '.' + f[:-3]
  200. if module not in testslist:
  201. testslist.append(module)
  202. for p in bbpath:
  203. testpath = os.path.join(p, 'lib', 'oeqa', type)
  204. bb.debug(2, 'Searching for tests in %s' % testpath)
  205. if os.path.exists(testpath):
  206. add_auto_list(testpath)
  207. return testslist
  208. def getTestModules(self):
  209. """
  210. Returns all the test modules in the testlist.
  211. """
  212. import pkgutil
  213. modules = []
  214. for test in self.testslist:
  215. if re.search("\w+\.\w+\.test_\S+", test):
  216. test = '.'.join(t.split('.')[:3])
  217. module = pkgutil.get_loader(test)
  218. modules.append(module)
  219. return modules
  220. def getModulefromID(self, test_id):
  221. """
  222. Returns the test module based on a test id.
  223. """
  224. module_name = ".".join(test_id.split(".")[:3])
  225. modules = self.getTestModules()
  226. for module in modules:
  227. if module.name == module_name:
  228. return module
  229. return None
  230. def getTests(self, test):
  231. '''Return all individual tests executed when running the suite.'''
  232. # Unfortunately unittest does not have an API for this, so we have
  233. # to rely on implementation details. This only needs to work
  234. # for TestSuite containing TestCase.
  235. method = getattr(test, '_testMethodName', None)
  236. if method:
  237. # leaf case: a TestCase
  238. yield test
  239. else:
  240. # Look into TestSuite.
  241. tests = getattr(test, '_tests', [])
  242. for t1 in tests:
  243. for t2 in self.getTests(t1):
  244. yield t2
  245. def loadTests(self):
  246. setattr(oeTest, "tc", self)
  247. testloader = unittest.TestLoader()
  248. testloader.sortTestMethodsUsing = None
  249. suites = [testloader.loadTestsFromName(name) for name in self.testslist]
  250. suites = filterByTagExp(suites, getattr(self, "tagexp", None))
  251. # Determine dependencies between suites by looking for @skipUnlessPassed
  252. # method annotations. Suite A depends on suite B if any method in A
  253. # depends on a method on B.
  254. for suite in suites:
  255. suite.dependencies = []
  256. suite.depth = 0
  257. for test in self.getTests(suite):
  258. methodname = getattr(test, '_testMethodName', None)
  259. if methodname:
  260. method = getattr(test, methodname)
  261. depends_on = getattr(method, '_depends_on', None)
  262. if depends_on:
  263. for dep_suite in suites:
  264. if depends_on in [getattr(t, '_testMethodName', None) for t in self.getTests(dep_suite)]:
  265. if dep_suite not in suite.dependencies and \
  266. dep_suite is not suite:
  267. suite.dependencies.append(dep_suite)
  268. break
  269. else:
  270. logger.warning("Test %s was declared as @skipUnlessPassed('%s') but that test is either not defined or not active. Will run the test anyway." %
  271. (test, depends_on))
  272. # Use brute-force topological sort to determine ordering. Sort by
  273. # depth (higher depth = must run later), with original ordering to
  274. # break ties.
  275. def set_suite_depth(suite):
  276. for dep in suite.dependencies:
  277. new_depth = set_suite_depth(dep) + 1
  278. if new_depth > suite.depth:
  279. suite.depth = new_depth
  280. return suite.depth
  281. for index, suite in enumerate(suites):
  282. set_suite_depth(suite)
  283. suite.index = index
  284. def cmp(a, b):
  285. return (a > b) - (a < b)
  286. def cmpfunc(a, b):
  287. return cmp((a.depth, a.index), (b.depth, b.index))
  288. suites.sort(key=functools.cmp_to_key(cmpfunc))
  289. self.suite = testloader.suiteClass(suites)
  290. return self.suite
  291. def runTests(self):
  292. logger.info("Test modules %s" % self.testslist)
  293. if hasattr(self, "tagexp") and self.tagexp:
  294. logger.info("Filter test cases by tags: %s" % self.tagexp)
  295. logger.info("Found %s tests" % self.suite.countTestCases())
  296. runner = unittest.TextTestRunner(verbosity=2)
  297. if 'bb' in sys.modules:
  298. runner.stream.write = custom_verbose
  299. return runner.run(self.suite)
  300. class RuntimeTestContext(TestContext):
  301. def __init__(self, d, target, exported=False):
  302. super(RuntimeTestContext, self).__init__(d, exported)
  303. self.target = target
  304. self.pkgmanifest = {}
  305. manifest = os.path.join(d.getVar("DEPLOY_DIR_IMAGE"),
  306. d.getVar("IMAGE_LINK_NAME") + ".manifest")
  307. nomanifest = d.getVar("IMAGE_NO_MANIFEST")
  308. if nomanifest is None or nomanifest != "1":
  309. try:
  310. with open(manifest) as f:
  311. for line in f:
  312. (pkg, arch, version) = line.strip().split()
  313. self.pkgmanifest[pkg] = (version, arch)
  314. except IOError as e:
  315. bb.fatal("No package manifest file found. Did you build the image?\n%s" % e)
  316. def _get_test_namespace(self):
  317. return "runtime"
  318. def _get_test_suites(self):
  319. testsuites = []
  320. manifests = (self.d.getVar("TEST_SUITES_MANIFEST") or '').split()
  321. if manifests:
  322. for manifest in manifests:
  323. testsuites.extend(self._read_testlist(manifest,
  324. self.d.getVar("TOPDIR")).split())
  325. else:
  326. testsuites = self.d.getVar("TEST_SUITES").split()
  327. return testsuites
  328. def _get_test_suites_required(self):
  329. return [t for t in self.d.getVar("TEST_SUITES").split() if t != "auto"]
  330. def loadTests(self):
  331. super(RuntimeTestContext, self).loadTests()
  332. if oeTest.hasPackage("procps"):
  333. oeRuntimeTest.pscmd = "ps -ef"
  334. def extract_packages(self):
  335. """
  336. Find packages that will be needed during runtime.
  337. """
  338. modules = self.getTestModules()
  339. bbpaths = self.d.getVar("BBPATH").split(":")
  340. shutil.rmtree(self.d.getVar("TEST_EXTRACTED_DIR"))
  341. shutil.rmtree(self.d.getVar("TEST_PACKAGED_DIR"))
  342. for module in modules:
  343. json_file = self._getJsonFile(module)
  344. if json_file:
  345. needed_packages = self._getNeededPackages(json_file)
  346. self._perform_package_extraction(needed_packages)
  347. def _perform_package_extraction(self, needed_packages):
  348. """
  349. Extract packages that will be needed during runtime.
  350. """
  351. import oe.path
  352. extracted_path = self.d.getVar("TEST_EXTRACTED_DIR")
  353. packaged_path = self.d.getVar("TEST_PACKAGED_DIR")
  354. for key,value in needed_packages.items():
  355. packages = ()
  356. if isinstance(value, dict):
  357. packages = (value, )
  358. elif isinstance(value, list):
  359. packages = value
  360. else:
  361. bb.fatal("Failed to process needed packages for %s; "
  362. "Value must be a dict or list" % key)
  363. for package in packages:
  364. pkg = package["pkg"]
  365. rm = package.get("rm", False)
  366. extract = package.get("extract", True)
  367. if extract:
  368. dst_dir = os.path.join(extracted_path, pkg)
  369. else:
  370. dst_dir = os.path.join(packaged_path)
  371. # Extract package and copy it to TEST_EXTRACTED_DIR
  372. pkg_dir = self._extract_in_tmpdir(pkg)
  373. if extract:
  374. # Same package used for more than one test,
  375. # don't need to extract again.
  376. if os.path.exists(dst_dir):
  377. continue
  378. oe.path.copytree(pkg_dir, dst_dir)
  379. shutil.rmtree(pkg_dir)
  380. # Copy package to TEST_PACKAGED_DIR
  381. else:
  382. self._copy_package(pkg)
  383. def _getJsonFile(self, module):
  384. """
  385. Returns the path of the JSON file for a module, empty if doesn't exitst.
  386. """
  387. module_file = module.path
  388. json_file = "%s.json" % module_file.rsplit(".", 1)[0]
  389. if os.path.isfile(module_file) and os.path.isfile(json_file):
  390. return json_file
  391. else:
  392. return ""
  393. def _getNeededPackages(self, json_file, test=None):
  394. """
  395. Returns a dict with needed packages based on a JSON file.
  396. If a test is specified it will return the dict just for that test.
  397. """
  398. import json
  399. needed_packages = {}
  400. with open(json_file) as f:
  401. test_packages = json.load(f)
  402. for key,value in test_packages.items():
  403. needed_packages[key] = value
  404. if test:
  405. if test in needed_packages:
  406. needed_packages = needed_packages[test]
  407. else:
  408. needed_packages = {}
  409. return needed_packages
  410. def _extract_in_tmpdir(self, pkg):
  411. """"
  412. Returns path to a temp directory where the package was
  413. extracted without dependencies.
  414. """
  415. from oeqa.utils.package_manager import get_package_manager
  416. pkg_path = os.path.join(self.d.getVar("TEST_INSTALL_TMP_DIR"), pkg)
  417. pm = get_package_manager(self.d, pkg_path)
  418. extract_dir = pm.extract(pkg)
  419. shutil.rmtree(pkg_path)
  420. return extract_dir
  421. def _copy_package(self, pkg):
  422. """
  423. Copy the RPM, DEB or IPK package to dst_dir
  424. """
  425. from oeqa.utils.package_manager import get_package_manager
  426. pkg_path = os.path.join(self.d.getVar("TEST_INSTALL_TMP_DIR"), pkg)
  427. dst_dir = self.d.getVar("TEST_PACKAGED_DIR")
  428. pm = get_package_manager(self.d, pkg_path)
  429. pkg_info = pm.package_info(pkg)
  430. file_path = pkg_info[pkg]["filepath"]
  431. shutil.copy2(file_path, dst_dir)
  432. shutil.rmtree(pkg_path)
  433. def install_uninstall_packages(self, test_id, pkg_dir, install):
  434. """
  435. Check if the test requires a package and Install/Uninstall it in the DUT
  436. """
  437. test = test_id.split(".")[4]
  438. module = self.getModulefromID(test_id)
  439. json = self._getJsonFile(module)
  440. if json:
  441. needed_packages = self._getNeededPackages(json, test)
  442. if needed_packages:
  443. self._install_uninstall_packages(needed_packages, pkg_dir, install)
  444. def _install_uninstall_packages(self, needed_packages, pkg_dir, install=True):
  445. """
  446. Install/Uninstall packages in the DUT without using a package manager
  447. """
  448. if isinstance(needed_packages, dict):
  449. packages = [needed_packages]
  450. elif isinstance(needed_packages, list):
  451. packages = needed_packages
  452. for package in packages:
  453. pkg = package["pkg"]
  454. rm = package.get("rm", False)
  455. extract = package.get("extract", True)
  456. src_dir = os.path.join(pkg_dir, pkg)
  457. # Install package
  458. if install and extract:
  459. self.target.connection.copy_dir_to(src_dir, "/")
  460. # Uninstall package
  461. elif not install and rm:
  462. self.target.connection.delete_dir_structure(src_dir, "/")
  463. class ImageTestContext(RuntimeTestContext):
  464. def __init__(self, d, target, host_dumper):
  465. super(ImageTestContext, self).__init__(d, target)
  466. self.tagexp = d.getVar("TEST_SUITES_TAGS")
  467. self.host_dumper = host_dumper
  468. self.sigterm = False
  469. self.origsigtermhandler = signal.getsignal(signal.SIGTERM)
  470. signal.signal(signal.SIGTERM, self._sigterm_exception)
  471. def _sigterm_exception(self, signum, stackframe):
  472. bb.warn("TestImage received SIGTERM, shutting down...")
  473. self.sigterm = True
  474. self.target.stop()
  475. def install_uninstall_packages(self, test_id, install=True):
  476. """
  477. Check if the test requires a package and Install/Uninstall it in the DUT
  478. """
  479. pkg_dir = self.d.getVar("TEST_EXTRACTED_DIR")
  480. super(ImageTestContext, self).install_uninstall_packages(test_id, pkg_dir, install)
  481. class ExportTestContext(RuntimeTestContext):
  482. def __init__(self, d, target, exported=False, parsedArgs={}):
  483. """
  484. This class is used when exporting tests and when are executed outside OE environment.
  485. parsedArgs can contain the following:
  486. - tag: Filter test by tag.
  487. """
  488. super(ExportTestContext, self).__init__(d, target, exported)
  489. tag = parsedArgs.get("tag", None)
  490. self.tagexp = tag if tag != None else d.getVar("TEST_SUITES_TAGS")
  491. self.sigterm = None
  492. def install_uninstall_packages(self, test_id, install=True):
  493. """
  494. Check if the test requires a package and Install/Uninstall it in the DUT
  495. """
  496. export_dir = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
  497. extracted_dir = self.d.getVar("TEST_EXPORT_EXTRACTED_DIR")
  498. pkg_dir = os.path.join(export_dir, extracted_dir)
  499. super(ExportTestContext, self).install_uninstall_packages(test_id, pkg_dir, install)