concurrencytest.py 10 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264
  1. #!/usr/bin/env python3
  2. #
  3. # SPDX-License-Identifier: GPL-2.0-or-later
  4. #
  5. # Modified for use in OE by Richard Purdie, 2018
  6. #
  7. # Modified by: Corey Goldberg, 2013
  8. # License: GPLv2+
  9. #
  10. # Original code from:
  11. # Bazaar (bzrlib.tests.__init__.py, v2.6, copied Jun 01 2013)
  12. # Copyright (C) 2005-2011 Canonical Ltd
  13. # License: GPLv2+
  14. import os
  15. import sys
  16. import traceback
  17. import unittest
  18. import subprocess
  19. import testtools
  20. import threading
  21. import time
  22. import io
  23. from queue import Queue
  24. from itertools import cycle
  25. from subunit import ProtocolTestCase, TestProtocolClient
  26. from subunit.test_results import AutoTimingTestResultDecorator
  27. from testtools import ThreadsafeForwardingResult, iterate_tests
  28. from oeqa.utils.commands import get_test_layer
  29. import bb.utils
  30. import oe.path
  31. _all__ = [
  32. 'ConcurrentTestSuite',
  33. 'fork_for_tests',
  34. 'partition_tests',
  35. ]
  36. #
  37. # Patch the version from testtools to allow access to _test_start and allow
  38. # computation of timing information and threading progress
  39. #
  40. class BBThreadsafeForwardingResult(ThreadsafeForwardingResult):
  41. def __init__(self, target, semaphore, threadnum, totalinprocess, totaltests):
  42. super(BBThreadsafeForwardingResult, self).__init__(target, semaphore)
  43. self.threadnum = threadnum
  44. self.totalinprocess = totalinprocess
  45. self.totaltests = totaltests
  46. def _add_result_with_semaphore(self, method, test, *args, **kwargs):
  47. self.semaphore.acquire()
  48. try:
  49. self.result.starttime[test.id()] = self._test_start.timestamp()
  50. self.result.threadprogress[self.threadnum].append(test.id())
  51. totalprogress = sum(len(x) for x in self.result.threadprogress.values())
  52. self.result.progressinfo[test.id()] = "%s: %s/%s %s/%s (%ss) (%s)" % (
  53. self.threadnum,
  54. len(self.result.threadprogress[self.threadnum]),
  55. self.totalinprocess,
  56. totalprogress,
  57. self.totaltests,
  58. "{0:.2f}".format(time.time()-self._test_start.timestamp()),
  59. test.id())
  60. finally:
  61. self.semaphore.release()
  62. super(BBThreadsafeForwardingResult, self)._add_result_with_semaphore(method, test, *args, **kwargs)
  63. #
  64. # A dummy structure to add to io.StringIO so that the .buffer object
  65. # is available and accepts writes. This allows unittest with buffer=True
  66. # to interact ok with subunit which wants to access sys.stdout.buffer.
  67. #
  68. class dummybuf(object):
  69. def __init__(self, parent):
  70. self.p = parent
  71. def write(self, data):
  72. self.p.write(data.decode("utf-8"))
  73. #
  74. # Taken from testtools.ConncurrencyTestSuite but modified for OE use
  75. #
  76. class ConcurrentTestSuite(unittest.TestSuite):
  77. def __init__(self, suite, processes):
  78. super(ConcurrentTestSuite, self).__init__([suite])
  79. self.processes = processes
  80. def run(self, result):
  81. tests, totaltests = fork_for_tests(self.processes, self)
  82. try:
  83. threads = {}
  84. queue = Queue()
  85. semaphore = threading.Semaphore(1)
  86. result.threadprogress = {}
  87. for i, (test, testnum) in enumerate(tests):
  88. result.threadprogress[i] = []
  89. process_result = BBThreadsafeForwardingResult(result, semaphore, i, testnum, totaltests)
  90. # Force buffering of stdout/stderr so the console doesn't get corrupted by test output
  91. # as per default in parent code
  92. process_result.buffer = True
  93. # We have to add a buffer object to stdout to keep subunit happy
  94. process_result._stderr_buffer = io.StringIO()
  95. process_result._stderr_buffer.buffer = dummybuf(process_result._stderr_buffer)
  96. process_result._stdout_buffer = io.StringIO()
  97. process_result._stdout_buffer.buffer = dummybuf(process_result._stdout_buffer)
  98. reader_thread = threading.Thread(
  99. target=self._run_test, args=(test, process_result, queue))
  100. threads[test] = reader_thread, process_result
  101. reader_thread.start()
  102. while threads:
  103. finished_test = queue.get()
  104. threads[finished_test][0].join()
  105. del threads[finished_test]
  106. except:
  107. for thread, process_result in threads.values():
  108. process_result.stop()
  109. raise
  110. finally:
  111. for test in tests:
  112. test[0]._stream.close()
  113. def _run_test(self, test, process_result, queue):
  114. try:
  115. try:
  116. test.run(process_result)
  117. except Exception:
  118. # The run logic itself failed
  119. case = testtools.ErrorHolder(
  120. "broken-runner",
  121. error=sys.exc_info())
  122. case.run(process_result)
  123. finally:
  124. queue.put(test)
  125. def removebuilddir(d):
  126. delay = 5
  127. while delay and os.path.exists(d + "/bitbake.lock"):
  128. time.sleep(1)
  129. delay = delay - 1
  130. bb.utils.prunedir(d)
  131. def fork_for_tests(concurrency_num, suite):
  132. result = []
  133. if 'BUILDDIR' in os.environ:
  134. selftestdir = get_test_layer()
  135. test_blocks = partition_tests(suite, concurrency_num)
  136. # Clear the tests from the original suite so it doesn't keep them alive
  137. suite._tests[:] = []
  138. totaltests = sum(len(x) for x in test_blocks)
  139. for process_tests in test_blocks:
  140. numtests = len(process_tests)
  141. process_suite = unittest.TestSuite(process_tests)
  142. # Also clear each split list so new suite has only reference
  143. process_tests[:] = []
  144. c2pread, c2pwrite = os.pipe()
  145. # Clear buffers before fork to avoid duplicate output
  146. sys.stdout.flush()
  147. sys.stderr.flush()
  148. pid = os.fork()
  149. if pid == 0:
  150. ourpid = os.getpid()
  151. try:
  152. newbuilddir = None
  153. stream = os.fdopen(c2pwrite, 'wb', 1)
  154. os.close(c2pread)
  155. # Create a new separate BUILDDIR for each group of tests
  156. if 'BUILDDIR' in os.environ:
  157. builddir = os.environ['BUILDDIR']
  158. newbuilddir = builddir + "-st-" + str(ourpid)
  159. newselftestdir = newbuilddir + "/meta-selftest"
  160. bb.utils.mkdirhier(newbuilddir)
  161. oe.path.copytree(builddir + "/conf", newbuilddir + "/conf")
  162. oe.path.copytree(builddir + "/cache", newbuilddir + "/cache")
  163. oe.path.copytree(selftestdir, newselftestdir)
  164. for e in os.environ:
  165. if builddir in os.environ[e]:
  166. os.environ[e] = os.environ[e].replace(builddir, newbuilddir)
  167. subprocess.check_output("git init; git add *; git commit -a -m 'initial'", cwd=newselftestdir, shell=True)
  168. # Tried to used bitbake-layers add/remove but it requires recipe parsing and hence is too slow
  169. subprocess.check_output("sed %s/conf/bblayers.conf -i -e 's#%s#%s#g'" % (newbuilddir, selftestdir, newselftestdir), cwd=newbuilddir, shell=True)
  170. os.chdir(newbuilddir)
  171. for t in process_suite:
  172. if not hasattr(t, "tc"):
  173. continue
  174. cp = t.tc.config_paths
  175. for p in cp:
  176. if selftestdir in cp[p] and newselftestdir not in cp[p]:
  177. cp[p] = cp[p].replace(selftestdir, newselftestdir)
  178. if builddir in cp[p] and newbuilddir not in cp[p]:
  179. cp[p] = cp[p].replace(builddir, newbuilddir)
  180. # Leave stderr and stdout open so we can see test noise
  181. # Close stdin so that the child goes away if it decides to
  182. # read from stdin (otherwise its a roulette to see what
  183. # child actually gets keystrokes for pdb etc).
  184. newsi = os.open(os.devnull, os.O_RDWR)
  185. os.dup2(newsi, sys.stdin.fileno())
  186. subunit_client = TestProtocolClient(stream)
  187. # Force buffering of stdout/stderr so the console doesn't get corrupted by test output
  188. # as per default in parent code
  189. subunit_client.buffer = True
  190. subunit_result = AutoTimingTestResultDecorator(subunit_client)
  191. process_suite.run(subunit_result)
  192. if ourpid != os.getpid():
  193. os._exit(0)
  194. if newbuilddir:
  195. removebuilddir(newbuilddir)
  196. except:
  197. # Don't do anything with process children
  198. if ourpid != os.getpid():
  199. os._exit(1)
  200. # Try and report traceback on stream, but exit with error
  201. # even if stream couldn't be created or something else
  202. # goes wrong. The traceback is formatted to a string and
  203. # written in one go to avoid interleaving lines from
  204. # multiple failing children.
  205. try:
  206. stream.write(traceback.format_exc().encode('utf-8'))
  207. except:
  208. sys.stderr.write(traceback.format_exc())
  209. finally:
  210. if newbuilddir:
  211. removebuilddir(newbuilddir)
  212. stream.flush()
  213. os._exit(1)
  214. stream.flush()
  215. os._exit(0)
  216. else:
  217. os.close(c2pwrite)
  218. stream = os.fdopen(c2pread, 'rb', 1)
  219. test = ProtocolTestCase(stream)
  220. result.append((test, numtests))
  221. return result, totaltests
  222. def partition_tests(suite, count):
  223. # Keep tests from the same class together but allow tests from modules
  224. # to go to different processes to aid parallelisation.
  225. modules = {}
  226. for test in iterate_tests(suite):
  227. m = test.__module__ + "." + test.__class__.__name__
  228. if m not in modules:
  229. modules[m] = []
  230. modules[m].append(test)
  231. # Simply divide the test blocks between the available processes
  232. partitions = [list() for _ in range(count)]
  233. for partition, m in zip(cycle(partitions), modules):
  234. partition.extend(modules[m])
  235. # No point in empty threads so drop them
  236. return [p for p in partitions if p]