builderthread.py 21 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500
  1. # SPDX-License-Identifier: GPL-2.0+
  2. # Copyright (c) 2014 Google, Inc
  3. #
  4. import errno
  5. import glob
  6. import os
  7. import shutil
  8. import sys
  9. import threading
  10. import command
  11. import gitutil
  12. RETURN_CODE_RETRY = -1
  13. def Mkdir(dirname, parents = False):
  14. """Make a directory if it doesn't already exist.
  15. Args:
  16. dirname: Directory to create
  17. """
  18. try:
  19. if parents:
  20. os.makedirs(dirname)
  21. else:
  22. os.mkdir(dirname)
  23. except OSError as err:
  24. if err.errno == errno.EEXIST:
  25. if os.path.realpath('.') == os.path.realpath(dirname):
  26. print("Cannot create the current working directory '%s'!" % dirname)
  27. sys.exit(1)
  28. pass
  29. else:
  30. raise
  31. class BuilderJob:
  32. """Holds information about a job to be performed by a thread
  33. Members:
  34. board: Board object to build
  35. commits: List of commit options to build.
  36. """
  37. def __init__(self):
  38. self.board = None
  39. self.commits = []
  40. class ResultThread(threading.Thread):
  41. """This thread processes results from builder threads.
  42. It simply passes the results on to the builder. There is only one
  43. result thread, and this helps to serialise the build output.
  44. """
  45. def __init__(self, builder):
  46. """Set up a new result thread
  47. Args:
  48. builder: Builder which will be sent each result
  49. """
  50. threading.Thread.__init__(self)
  51. self.builder = builder
  52. def run(self):
  53. """Called to start up the result thread.
  54. We collect the next result job and pass it on to the build.
  55. """
  56. while True:
  57. result = self.builder.out_queue.get()
  58. self.builder.ProcessResult(result)
  59. self.builder.out_queue.task_done()
  60. class BuilderThread(threading.Thread):
  61. """This thread builds U-Boot for a particular board.
  62. An input queue provides each new job. We run 'make' to build U-Boot
  63. and then pass the results on to the output queue.
  64. Members:
  65. builder: The builder which contains information we might need
  66. thread_num: Our thread number (0-n-1), used to decide on a
  67. temporary directory
  68. """
  69. def __init__(self, builder, thread_num, incremental, per_board_out_dir):
  70. """Set up a new builder thread"""
  71. threading.Thread.__init__(self)
  72. self.builder = builder
  73. self.thread_num = thread_num
  74. self.incremental = incremental
  75. self.per_board_out_dir = per_board_out_dir
  76. def Make(self, commit, brd, stage, cwd, *args, **kwargs):
  77. """Run 'make' on a particular commit and board.
  78. The source code will already be checked out, so the 'commit'
  79. argument is only for information.
  80. Args:
  81. commit: Commit object that is being built
  82. brd: Board object that is being built
  83. stage: Stage of the build. Valid stages are:
  84. mrproper - can be called to clean source
  85. config - called to configure for a board
  86. build - the main make invocation - it does the build
  87. args: A list of arguments to pass to 'make'
  88. kwargs: A list of keyword arguments to pass to command.RunPipe()
  89. Returns:
  90. CommandResult object
  91. """
  92. return self.builder.do_make(commit, brd, stage, cwd, *args,
  93. **kwargs)
  94. def RunCommit(self, commit_upto, brd, work_dir, do_config, config_only,
  95. force_build, force_build_failures):
  96. """Build a particular commit.
  97. If the build is already done, and we are not forcing a build, we skip
  98. the build and just return the previously-saved results.
  99. Args:
  100. commit_upto: Commit number to build (0...n-1)
  101. brd: Board object to build
  102. work_dir: Directory to which the source will be checked out
  103. do_config: True to run a make <board>_defconfig on the source
  104. config_only: Only configure the source, do not build it
  105. force_build: Force a build even if one was previously done
  106. force_build_failures: Force a bulid if the previous result showed
  107. failure
  108. Returns:
  109. tuple containing:
  110. - CommandResult object containing the results of the build
  111. - boolean indicating whether 'make config' is still needed
  112. """
  113. # Create a default result - it will be overwritte by the call to
  114. # self.Make() below, in the event that we do a build.
  115. result = command.CommandResult()
  116. result.return_code = 0
  117. if self.builder.in_tree:
  118. out_dir = work_dir
  119. else:
  120. if self.per_board_out_dir:
  121. out_rel_dir = os.path.join('..', brd.target)
  122. else:
  123. out_rel_dir = 'build'
  124. out_dir = os.path.join(work_dir, out_rel_dir)
  125. # Check if the job was already completed last time
  126. done_file = self.builder.GetDoneFile(commit_upto, brd.target)
  127. result.already_done = os.path.exists(done_file)
  128. will_build = (force_build or force_build_failures or
  129. not result.already_done)
  130. if result.already_done:
  131. # Get the return code from that build and use it
  132. with open(done_file, 'r') as fd:
  133. try:
  134. result.return_code = int(fd.readline())
  135. except ValueError:
  136. # The file may be empty due to running out of disk space.
  137. # Try a rebuild
  138. result.return_code = RETURN_CODE_RETRY
  139. # Check the signal that the build needs to be retried
  140. if result.return_code == RETURN_CODE_RETRY:
  141. will_build = True
  142. elif will_build:
  143. err_file = self.builder.GetErrFile(commit_upto, brd.target)
  144. if os.path.exists(err_file) and os.stat(err_file).st_size:
  145. result.stderr = 'bad'
  146. elif not force_build:
  147. # The build passed, so no need to build it again
  148. will_build = False
  149. if will_build:
  150. # We are going to have to build it. First, get a toolchain
  151. if not self.toolchain:
  152. try:
  153. self.toolchain = self.builder.toolchains.Select(brd.arch)
  154. except ValueError as err:
  155. result.return_code = 10
  156. result.stdout = ''
  157. result.stderr = str(err)
  158. # TODO(sjg@chromium.org): This gets swallowed, but needs
  159. # to be reported.
  160. if self.toolchain:
  161. # Checkout the right commit
  162. if self.builder.commits:
  163. commit = self.builder.commits[commit_upto]
  164. if self.builder.checkout:
  165. git_dir = os.path.join(work_dir, '.git')
  166. gitutil.Checkout(commit.hash, git_dir, work_dir,
  167. force=True)
  168. else:
  169. commit = 'current'
  170. # Set up the environment and command line
  171. env = self.toolchain.MakeEnvironment(self.builder.full_path)
  172. Mkdir(out_dir)
  173. args = []
  174. cwd = work_dir
  175. src_dir = os.path.realpath(work_dir)
  176. if not self.builder.in_tree:
  177. if commit_upto is None:
  178. # In this case we are building in the original source
  179. # directory (i.e. the current directory where buildman
  180. # is invoked. The output directory is set to this
  181. # thread's selected work directory.
  182. #
  183. # Symlinks can confuse U-Boot's Makefile since
  184. # we may use '..' in our path, so remove them.
  185. out_dir = os.path.realpath(out_dir)
  186. args.append('O=%s' % out_dir)
  187. cwd = None
  188. src_dir = os.getcwd()
  189. else:
  190. args.append('O=%s' % out_rel_dir)
  191. if self.builder.verbose_build:
  192. args.append('V=1')
  193. else:
  194. args.append('-s')
  195. if self.builder.num_jobs is not None:
  196. args.extend(['-j', str(self.builder.num_jobs)])
  197. if self.builder.warnings_as_errors:
  198. args.append('KCFLAGS=-Werror')
  199. config_args = ['%s_defconfig' % brd.target]
  200. config_out = ''
  201. args.extend(self.builder.toolchains.GetMakeArguments(brd))
  202. args.extend(self.toolchain.MakeArgs())
  203. # If we need to reconfigure, do that now
  204. if do_config:
  205. config_out = ''
  206. if not self.incremental:
  207. result = self.Make(commit, brd, 'mrproper', cwd,
  208. 'mrproper', *args, env=env)
  209. config_out += result.combined
  210. result = self.Make(commit, brd, 'config', cwd,
  211. *(args + config_args), env=env)
  212. config_out += result.combined
  213. do_config = False # No need to configure next time
  214. if result.return_code == 0:
  215. if config_only:
  216. args.append('cfg')
  217. result = self.Make(commit, brd, 'build', cwd, *args,
  218. env=env)
  219. result.stderr = result.stderr.replace(src_dir + '/', '')
  220. if self.builder.verbose_build:
  221. result.stdout = config_out + result.stdout
  222. else:
  223. result.return_code = 1
  224. result.stderr = 'No tool chain for %s\n' % brd.arch
  225. result.already_done = False
  226. result.toolchain = self.toolchain
  227. result.brd = brd
  228. result.commit_upto = commit_upto
  229. result.out_dir = out_dir
  230. return result, do_config
  231. def _WriteResult(self, result, keep_outputs):
  232. """Write a built result to the output directory.
  233. Args:
  234. result: CommandResult object containing result to write
  235. keep_outputs: True to store the output binaries, False
  236. to delete them
  237. """
  238. # Fatal error
  239. if result.return_code < 0:
  240. return
  241. # If we think this might have been aborted with Ctrl-C, record the
  242. # failure but not that we are 'done' with this board. A retry may fix
  243. # it.
  244. maybe_aborted = result.stderr and 'No child processes' in result.stderr
  245. if result.already_done:
  246. return
  247. # Write the output and stderr
  248. output_dir = self.builder._GetOutputDir(result.commit_upto)
  249. Mkdir(output_dir)
  250. build_dir = self.builder.GetBuildDir(result.commit_upto,
  251. result.brd.target)
  252. Mkdir(build_dir)
  253. outfile = os.path.join(build_dir, 'log')
  254. with open(outfile, 'w') as fd:
  255. if result.stdout:
  256. fd.write(result.stdout)
  257. errfile = self.builder.GetErrFile(result.commit_upto,
  258. result.brd.target)
  259. if result.stderr:
  260. with open(errfile, 'w') as fd:
  261. fd.write(result.stderr)
  262. elif os.path.exists(errfile):
  263. os.remove(errfile)
  264. if result.toolchain:
  265. # Write the build result and toolchain information.
  266. done_file = self.builder.GetDoneFile(result.commit_upto,
  267. result.brd.target)
  268. with open(done_file, 'w') as fd:
  269. if maybe_aborted:
  270. # Special code to indicate we need to retry
  271. fd.write('%s' % RETURN_CODE_RETRY)
  272. else:
  273. fd.write('%s' % result.return_code)
  274. with open(os.path.join(build_dir, 'toolchain'), 'w') as fd:
  275. print('gcc', result.toolchain.gcc, file=fd)
  276. print('path', result.toolchain.path, file=fd)
  277. print('cross', result.toolchain.cross, file=fd)
  278. print('arch', result.toolchain.arch, file=fd)
  279. fd.write('%s' % result.return_code)
  280. # Write out the image and function size information and an objdump
  281. env = result.toolchain.MakeEnvironment(self.builder.full_path)
  282. with open(os.path.join(build_dir, 'env'), 'w') as fd:
  283. for var in sorted(env.keys()):
  284. print('%s="%s"' % (var, env[var]), file=fd)
  285. lines = []
  286. for fname in ['u-boot', 'spl/u-boot-spl']:
  287. cmd = ['%snm' % self.toolchain.cross, '--size-sort', fname]
  288. nm_result = command.RunPipe([cmd], capture=True,
  289. capture_stderr=True, cwd=result.out_dir,
  290. raise_on_error=False, env=env)
  291. if nm_result.stdout:
  292. nm = self.builder.GetFuncSizesFile(result.commit_upto,
  293. result.brd.target, fname)
  294. with open(nm, 'w') as fd:
  295. print(nm_result.stdout, end=' ', file=fd)
  296. cmd = ['%sobjdump' % self.toolchain.cross, '-h', fname]
  297. dump_result = command.RunPipe([cmd], capture=True,
  298. capture_stderr=True, cwd=result.out_dir,
  299. raise_on_error=False, env=env)
  300. rodata_size = ''
  301. if dump_result.stdout:
  302. objdump = self.builder.GetObjdumpFile(result.commit_upto,
  303. result.brd.target, fname)
  304. with open(objdump, 'w') as fd:
  305. print(dump_result.stdout, end=' ', file=fd)
  306. for line in dump_result.stdout.splitlines():
  307. fields = line.split()
  308. if len(fields) > 5 and fields[1] == '.rodata':
  309. rodata_size = fields[2]
  310. cmd = ['%ssize' % self.toolchain.cross, fname]
  311. size_result = command.RunPipe([cmd], capture=True,
  312. capture_stderr=True, cwd=result.out_dir,
  313. raise_on_error=False, env=env)
  314. if size_result.stdout:
  315. lines.append(size_result.stdout.splitlines()[1] + ' ' +
  316. rodata_size)
  317. # Extract the environment from U-Boot and dump it out
  318. cmd = ['%sobjcopy' % self.toolchain.cross, '-O', 'binary',
  319. '-j', '.rodata.default_environment',
  320. 'env/built-in.o', 'uboot.env']
  321. command.RunPipe([cmd], capture=True,
  322. capture_stderr=True, cwd=result.out_dir,
  323. raise_on_error=False, env=env)
  324. ubootenv = os.path.join(result.out_dir, 'uboot.env')
  325. self.CopyFiles(result.out_dir, build_dir, '', ['uboot.env'])
  326. # Write out the image sizes file. This is similar to the output
  327. # of binutil's 'size' utility, but it omits the header line and
  328. # adds an additional hex value at the end of each line for the
  329. # rodata size
  330. if len(lines):
  331. sizes = self.builder.GetSizesFile(result.commit_upto,
  332. result.brd.target)
  333. with open(sizes, 'w') as fd:
  334. print('\n'.join(lines), file=fd)
  335. # Write out the configuration files, with a special case for SPL
  336. for dirname in ['', 'spl', 'tpl']:
  337. self.CopyFiles(result.out_dir, build_dir, dirname, ['u-boot.cfg',
  338. 'spl/u-boot-spl.cfg', 'tpl/u-boot-tpl.cfg', '.config',
  339. 'include/autoconf.mk', 'include/generated/autoconf.h'])
  340. # Now write the actual build output
  341. if keep_outputs:
  342. self.CopyFiles(result.out_dir, build_dir, '', ['u-boot*', '*.bin',
  343. '*.map', '*.img', 'MLO', 'SPL', 'include/autoconf.mk',
  344. 'spl/u-boot-spl*'])
  345. def CopyFiles(self, out_dir, build_dir, dirname, patterns):
  346. """Copy files from the build directory to the output.
  347. Args:
  348. out_dir: Path to output directory containing the files
  349. build_dir: Place to copy the files
  350. dirname: Source directory, '' for normal U-Boot, 'spl' for SPL
  351. patterns: A list of filenames (strings) to copy, each relative
  352. to the build directory
  353. """
  354. for pattern in patterns:
  355. file_list = glob.glob(os.path.join(out_dir, dirname, pattern))
  356. for fname in file_list:
  357. target = os.path.basename(fname)
  358. if dirname:
  359. base, ext = os.path.splitext(target)
  360. if ext:
  361. target = '%s-%s%s' % (base, dirname, ext)
  362. shutil.copy(fname, os.path.join(build_dir, target))
  363. def RunJob(self, job):
  364. """Run a single job
  365. A job consists of a building a list of commits for a particular board.
  366. Args:
  367. job: Job to build
  368. """
  369. brd = job.board
  370. work_dir = self.builder.GetThreadDir(self.thread_num)
  371. self.toolchain = None
  372. if job.commits:
  373. # Run 'make board_defconfig' on the first commit
  374. do_config = True
  375. commit_upto = 0
  376. force_build = False
  377. for commit_upto in range(0, len(job.commits), job.step):
  378. result, request_config = self.RunCommit(commit_upto, brd,
  379. work_dir, do_config, self.builder.config_only,
  380. force_build or self.builder.force_build,
  381. self.builder.force_build_failures)
  382. failed = result.return_code or result.stderr
  383. did_config = do_config
  384. if failed and not do_config:
  385. # If our incremental build failed, try building again
  386. # with a reconfig.
  387. if self.builder.force_config_on_failure:
  388. result, request_config = self.RunCommit(commit_upto,
  389. brd, work_dir, True, False, True, False)
  390. did_config = True
  391. if not self.builder.force_reconfig:
  392. do_config = request_config
  393. # If we built that commit, then config is done. But if we got
  394. # an warning, reconfig next time to force it to build the same
  395. # files that created warnings this time. Otherwise an
  396. # incremental build may not build the same file, and we will
  397. # think that the warning has gone away.
  398. # We could avoid this by using -Werror everywhere...
  399. # For errors, the problem doesn't happen, since presumably
  400. # the build stopped and didn't generate output, so will retry
  401. # that file next time. So we could detect warnings and deal
  402. # with them specially here. For now, we just reconfigure if
  403. # anything goes work.
  404. # Of course this is substantially slower if there are build
  405. # errors/warnings (e.g. 2-3x slower even if only 10% of builds
  406. # have problems).
  407. if (failed and not result.already_done and not did_config and
  408. self.builder.force_config_on_failure):
  409. # If this build failed, try the next one with a
  410. # reconfigure.
  411. # Sometimes if the board_config.h file changes it can mess
  412. # with dependencies, and we get:
  413. # make: *** No rule to make target `include/autoconf.mk',
  414. # needed by `depend'.
  415. do_config = True
  416. force_build = True
  417. else:
  418. force_build = False
  419. if self.builder.force_config_on_failure:
  420. if failed:
  421. do_config = True
  422. result.commit_upto = commit_upto
  423. if result.return_code < 0:
  424. raise ValueError('Interrupt')
  425. # We have the build results, so output the result
  426. self._WriteResult(result, job.keep_outputs)
  427. self.builder.out_queue.put(result)
  428. else:
  429. # Just build the currently checked-out build
  430. result, request_config = self.RunCommit(None, brd, work_dir, True,
  431. self.builder.config_only, True,
  432. self.builder.force_build_failures)
  433. result.commit_upto = 0
  434. self._WriteResult(result, job.keep_outputs)
  435. self.builder.out_queue.put(result)
  436. def run(self):
  437. """Our thread's run function
  438. This thread picks a job from the queue, runs it, and then goes to the
  439. next job.
  440. """
  441. while True:
  442. job = self.builder.queue.get()
  443. self.RunJob(job)
  444. self.builder.queue.task_done()