builderthread.py 24 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563
  1. # SPDX-License-Identifier: GPL-2.0+
  2. # Copyright (c) 2014 Google, Inc
  3. #
  4. import errno
  5. import glob
  6. import os
  7. import shutil
  8. import sys
  9. import threading
  10. from patman import command
  11. from patman import gitutil
  12. RETURN_CODE_RETRY = -1
  13. BASE_ELF_FILENAMES = ['u-boot', 'spl/u-boot-spl', 'tpl/u-boot-tpl']
  14. def Mkdir(dirname, parents = False):
  15. """Make a directory if it doesn't already exist.
  16. Args:
  17. dirname: Directory to create
  18. """
  19. try:
  20. if parents:
  21. os.makedirs(dirname)
  22. else:
  23. os.mkdir(dirname)
  24. except OSError as err:
  25. if err.errno == errno.EEXIST:
  26. if os.path.realpath('.') == os.path.realpath(dirname):
  27. print("Cannot create the current working directory '%s'!" % dirname)
  28. sys.exit(1)
  29. pass
  30. else:
  31. raise
  32. class BuilderJob:
  33. """Holds information about a job to be performed by a thread
  34. Members:
  35. board: Board object to build
  36. commits: List of Commit objects to build
  37. keep_outputs: True to save build output files
  38. step: 1 to process every commit, n to process every nth commit
  39. work_in_output: Use the output directory as the work directory and
  40. don't write to a separate output directory.
  41. """
  42. def __init__(self):
  43. self.board = None
  44. self.commits = []
  45. self.keep_outputs = False
  46. self.step = 1
  47. self.work_in_output = False
  48. class ResultThread(threading.Thread):
  49. """This thread processes results from builder threads.
  50. It simply passes the results on to the builder. There is only one
  51. result thread, and this helps to serialise the build output.
  52. """
  53. def __init__(self, builder):
  54. """Set up a new result thread
  55. Args:
  56. builder: Builder which will be sent each result
  57. """
  58. threading.Thread.__init__(self)
  59. self.builder = builder
  60. def run(self):
  61. """Called to start up the result thread.
  62. We collect the next result job and pass it on to the build.
  63. """
  64. while True:
  65. result = self.builder.out_queue.get()
  66. self.builder.ProcessResult(result)
  67. self.builder.out_queue.task_done()
  68. class BuilderThread(threading.Thread):
  69. """This thread builds U-Boot for a particular board.
  70. An input queue provides each new job. We run 'make' to build U-Boot
  71. and then pass the results on to the output queue.
  72. Members:
  73. builder: The builder which contains information we might need
  74. thread_num: Our thread number (0-n-1), used to decide on a
  75. temporary directory. If this is -1 then there are no threads
  76. and we are the (only) main process
  77. mrproper: Use 'make mrproper' before each reconfigure
  78. per_board_out_dir: True to build in a separate persistent directory per
  79. board rather than a thread-specific directory
  80. test_exception: Used for testing; True to raise an exception instead of
  81. reporting the build result
  82. """
  83. def __init__(self, builder, thread_num, mrproper, per_board_out_dir,
  84. test_exception=False):
  85. """Set up a new builder thread"""
  86. threading.Thread.__init__(self)
  87. self.builder = builder
  88. self.thread_num = thread_num
  89. self.mrproper = mrproper
  90. self.per_board_out_dir = per_board_out_dir
  91. self.test_exception = test_exception
  92. def Make(self, commit, brd, stage, cwd, *args, **kwargs):
  93. """Run 'make' on a particular commit and board.
  94. The source code will already be checked out, so the 'commit'
  95. argument is only for information.
  96. Args:
  97. commit: Commit object that is being built
  98. brd: Board object that is being built
  99. stage: Stage of the build. Valid stages are:
  100. mrproper - can be called to clean source
  101. config - called to configure for a board
  102. build - the main make invocation - it does the build
  103. args: A list of arguments to pass to 'make'
  104. kwargs: A list of keyword arguments to pass to command.RunPipe()
  105. Returns:
  106. CommandResult object
  107. """
  108. return self.builder.do_make(commit, brd, stage, cwd, *args,
  109. **kwargs)
  110. def RunCommit(self, commit_upto, brd, work_dir, do_config, config_only,
  111. force_build, force_build_failures, work_in_output):
  112. """Build a particular commit.
  113. If the build is already done, and we are not forcing a build, we skip
  114. the build and just return the previously-saved results.
  115. Args:
  116. commit_upto: Commit number to build (0...n-1)
  117. brd: Board object to build
  118. work_dir: Directory to which the source will be checked out
  119. do_config: True to run a make <board>_defconfig on the source
  120. config_only: Only configure the source, do not build it
  121. force_build: Force a build even if one was previously done
  122. force_build_failures: Force a bulid if the previous result showed
  123. failure
  124. work_in_output: Use the output directory as the work directory and
  125. don't write to a separate output directory.
  126. Returns:
  127. tuple containing:
  128. - CommandResult object containing the results of the build
  129. - boolean indicating whether 'make config' is still needed
  130. """
  131. # Create a default result - it will be overwritte by the call to
  132. # self.Make() below, in the event that we do a build.
  133. result = command.CommandResult()
  134. result.return_code = 0
  135. if work_in_output or self.builder.in_tree:
  136. out_dir = work_dir
  137. else:
  138. if self.per_board_out_dir:
  139. out_rel_dir = os.path.join('..', brd.target)
  140. else:
  141. out_rel_dir = 'build'
  142. out_dir = os.path.join(work_dir, out_rel_dir)
  143. # Check if the job was already completed last time
  144. done_file = self.builder.GetDoneFile(commit_upto, brd.target)
  145. result.already_done = os.path.exists(done_file)
  146. will_build = (force_build or force_build_failures or
  147. not result.already_done)
  148. if result.already_done:
  149. # Get the return code from that build and use it
  150. with open(done_file, 'r') as fd:
  151. try:
  152. result.return_code = int(fd.readline())
  153. except ValueError:
  154. # The file may be empty due to running out of disk space.
  155. # Try a rebuild
  156. result.return_code = RETURN_CODE_RETRY
  157. # Check the signal that the build needs to be retried
  158. if result.return_code == RETURN_CODE_RETRY:
  159. will_build = True
  160. elif will_build:
  161. err_file = self.builder.GetErrFile(commit_upto, brd.target)
  162. if os.path.exists(err_file) and os.stat(err_file).st_size:
  163. result.stderr = 'bad'
  164. elif not force_build:
  165. # The build passed, so no need to build it again
  166. will_build = False
  167. if will_build:
  168. # We are going to have to build it. First, get a toolchain
  169. if not self.toolchain:
  170. try:
  171. self.toolchain = self.builder.toolchains.Select(brd.arch)
  172. except ValueError as err:
  173. result.return_code = 10
  174. result.stdout = ''
  175. result.stderr = str(err)
  176. # TODO(sjg@chromium.org): This gets swallowed, but needs
  177. # to be reported.
  178. if self.toolchain:
  179. # Checkout the right commit
  180. if self.builder.commits:
  181. commit = self.builder.commits[commit_upto]
  182. if self.builder.checkout:
  183. git_dir = os.path.join(work_dir, '.git')
  184. gitutil.Checkout(commit.hash, git_dir, work_dir,
  185. force=True)
  186. else:
  187. commit = 'current'
  188. # Set up the environment and command line
  189. env = self.toolchain.MakeEnvironment(self.builder.full_path)
  190. Mkdir(out_dir)
  191. args = []
  192. cwd = work_dir
  193. src_dir = os.path.realpath(work_dir)
  194. if not self.builder.in_tree:
  195. if commit_upto is None:
  196. # In this case we are building in the original source
  197. # directory (i.e. the current directory where buildman
  198. # is invoked. The output directory is set to this
  199. # thread's selected work directory.
  200. #
  201. # Symlinks can confuse U-Boot's Makefile since
  202. # we may use '..' in our path, so remove them.
  203. out_dir = os.path.realpath(out_dir)
  204. args.append('O=%s' % out_dir)
  205. cwd = None
  206. src_dir = os.getcwd()
  207. else:
  208. args.append('O=%s' % out_rel_dir)
  209. if self.builder.verbose_build:
  210. args.append('V=1')
  211. else:
  212. args.append('-s')
  213. if self.builder.num_jobs is not None:
  214. args.extend(['-j', str(self.builder.num_jobs)])
  215. if self.builder.warnings_as_errors:
  216. args.append('KCFLAGS=-Werror')
  217. config_args = ['%s_defconfig' % brd.target]
  218. config_out = ''
  219. args.extend(self.builder.toolchains.GetMakeArguments(brd))
  220. args.extend(self.toolchain.MakeArgs())
  221. # Remove any output targets. Since we use a build directory that
  222. # was previously used by another board, it may have produced an
  223. # SPL image. If we don't remove it (i.e. see do_config and
  224. # self.mrproper below) then it will appear to be the output of
  225. # this build, even if it does not produce SPL images.
  226. build_dir = self.builder.GetBuildDir(commit_upto, brd.target)
  227. for elf in BASE_ELF_FILENAMES:
  228. fname = os.path.join(out_dir, elf)
  229. if os.path.exists(fname):
  230. os.remove(fname)
  231. # If we need to reconfigure, do that now
  232. if do_config:
  233. config_out = ''
  234. if self.mrproper:
  235. result = self.Make(commit, brd, 'mrproper', cwd,
  236. 'mrproper', *args, env=env)
  237. config_out += result.combined
  238. result = self.Make(commit, brd, 'config', cwd,
  239. *(args + config_args), env=env)
  240. config_out += result.combined
  241. do_config = False # No need to configure next time
  242. if result.return_code == 0:
  243. if config_only:
  244. args.append('cfg')
  245. result = self.Make(commit, brd, 'build', cwd, *args,
  246. env=env)
  247. result.stderr = result.stderr.replace(src_dir + '/', '')
  248. if self.builder.verbose_build:
  249. result.stdout = config_out + result.stdout
  250. else:
  251. result.return_code = 1
  252. result.stderr = 'No tool chain for %s\n' % brd.arch
  253. result.already_done = False
  254. result.toolchain = self.toolchain
  255. result.brd = brd
  256. result.commit_upto = commit_upto
  257. result.out_dir = out_dir
  258. return result, do_config
  259. def _WriteResult(self, result, keep_outputs, work_in_output):
  260. """Write a built result to the output directory.
  261. Args:
  262. result: CommandResult object containing result to write
  263. keep_outputs: True to store the output binaries, False
  264. to delete them
  265. work_in_output: Use the output directory as the work directory and
  266. don't write to a separate output directory.
  267. """
  268. # Fatal error
  269. if result.return_code < 0:
  270. return
  271. # If we think this might have been aborted with Ctrl-C, record the
  272. # failure but not that we are 'done' with this board. A retry may fix
  273. # it.
  274. maybe_aborted = result.stderr and 'No child processes' in result.stderr
  275. if result.already_done:
  276. return
  277. # Write the output and stderr
  278. output_dir = self.builder._GetOutputDir(result.commit_upto)
  279. Mkdir(output_dir)
  280. build_dir = self.builder.GetBuildDir(result.commit_upto,
  281. result.brd.target)
  282. Mkdir(build_dir)
  283. outfile = os.path.join(build_dir, 'log')
  284. with open(outfile, 'w') as fd:
  285. if result.stdout:
  286. fd.write(result.stdout)
  287. errfile = self.builder.GetErrFile(result.commit_upto,
  288. result.brd.target)
  289. if result.stderr:
  290. with open(errfile, 'w') as fd:
  291. fd.write(result.stderr)
  292. elif os.path.exists(errfile):
  293. os.remove(errfile)
  294. if result.toolchain:
  295. # Write the build result and toolchain information.
  296. done_file = self.builder.GetDoneFile(result.commit_upto,
  297. result.brd.target)
  298. with open(done_file, 'w') as fd:
  299. if maybe_aborted:
  300. # Special code to indicate we need to retry
  301. fd.write('%s' % RETURN_CODE_RETRY)
  302. else:
  303. fd.write('%s' % result.return_code)
  304. with open(os.path.join(build_dir, 'toolchain'), 'w') as fd:
  305. print('gcc', result.toolchain.gcc, file=fd)
  306. print('path', result.toolchain.path, file=fd)
  307. print('cross', result.toolchain.cross, file=fd)
  308. print('arch', result.toolchain.arch, file=fd)
  309. fd.write('%s' % result.return_code)
  310. # Write out the image and function size information and an objdump
  311. env = result.toolchain.MakeEnvironment(self.builder.full_path)
  312. with open(os.path.join(build_dir, 'out-env'), 'w',
  313. encoding='utf-8') as fd:
  314. for var in sorted(env.keys()):
  315. print('%s="%s"' % (var, env[var]), file=fd)
  316. lines = []
  317. for fname in BASE_ELF_FILENAMES:
  318. cmd = ['%snm' % self.toolchain.cross, '--size-sort', fname]
  319. nm_result = command.RunPipe([cmd], capture=True,
  320. capture_stderr=True, cwd=result.out_dir,
  321. raise_on_error=False, env=env)
  322. if nm_result.stdout:
  323. nm = self.builder.GetFuncSizesFile(result.commit_upto,
  324. result.brd.target, fname)
  325. with open(nm, 'w') as fd:
  326. print(nm_result.stdout, end=' ', file=fd)
  327. cmd = ['%sobjdump' % self.toolchain.cross, '-h', fname]
  328. dump_result = command.RunPipe([cmd], capture=True,
  329. capture_stderr=True, cwd=result.out_dir,
  330. raise_on_error=False, env=env)
  331. rodata_size = ''
  332. if dump_result.stdout:
  333. objdump = self.builder.GetObjdumpFile(result.commit_upto,
  334. result.brd.target, fname)
  335. with open(objdump, 'w') as fd:
  336. print(dump_result.stdout, end=' ', file=fd)
  337. for line in dump_result.stdout.splitlines():
  338. fields = line.split()
  339. if len(fields) > 5 and fields[1] == '.rodata':
  340. rodata_size = fields[2]
  341. cmd = ['%ssize' % self.toolchain.cross, fname]
  342. size_result = command.RunPipe([cmd], capture=True,
  343. capture_stderr=True, cwd=result.out_dir,
  344. raise_on_error=False, env=env)
  345. if size_result.stdout:
  346. lines.append(size_result.stdout.splitlines()[1] + ' ' +
  347. rodata_size)
  348. # Extract the environment from U-Boot and dump it out
  349. cmd = ['%sobjcopy' % self.toolchain.cross, '-O', 'binary',
  350. '-j', '.rodata.default_environment',
  351. 'env/built-in.o', 'uboot.env']
  352. command.RunPipe([cmd], capture=True,
  353. capture_stderr=True, cwd=result.out_dir,
  354. raise_on_error=False, env=env)
  355. ubootenv = os.path.join(result.out_dir, 'uboot.env')
  356. if not work_in_output:
  357. self.CopyFiles(result.out_dir, build_dir, '', ['uboot.env'])
  358. # Write out the image sizes file. This is similar to the output
  359. # of binutil's 'size' utility, but it omits the header line and
  360. # adds an additional hex value at the end of each line for the
  361. # rodata size
  362. if len(lines):
  363. sizes = self.builder.GetSizesFile(result.commit_upto,
  364. result.brd.target)
  365. with open(sizes, 'w') as fd:
  366. print('\n'.join(lines), file=fd)
  367. if not work_in_output:
  368. # Write out the configuration files, with a special case for SPL
  369. for dirname in ['', 'spl', 'tpl']:
  370. self.CopyFiles(
  371. result.out_dir, build_dir, dirname,
  372. ['u-boot.cfg', 'spl/u-boot-spl.cfg', 'tpl/u-boot-tpl.cfg',
  373. '.config', 'include/autoconf.mk',
  374. 'include/generated/autoconf.h'])
  375. # Now write the actual build output
  376. if keep_outputs:
  377. self.CopyFiles(
  378. result.out_dir, build_dir, '',
  379. ['u-boot*', '*.bin', '*.map', '*.img', 'MLO', 'SPL',
  380. 'include/autoconf.mk', 'spl/u-boot-spl*'])
  381. def CopyFiles(self, out_dir, build_dir, dirname, patterns):
  382. """Copy files from the build directory to the output.
  383. Args:
  384. out_dir: Path to output directory containing the files
  385. build_dir: Place to copy the files
  386. dirname: Source directory, '' for normal U-Boot, 'spl' for SPL
  387. patterns: A list of filenames (strings) to copy, each relative
  388. to the build directory
  389. """
  390. for pattern in patterns:
  391. file_list = glob.glob(os.path.join(out_dir, dirname, pattern))
  392. for fname in file_list:
  393. target = os.path.basename(fname)
  394. if dirname:
  395. base, ext = os.path.splitext(target)
  396. if ext:
  397. target = '%s-%s%s' % (base, dirname, ext)
  398. shutil.copy(fname, os.path.join(build_dir, target))
  399. def _SendResult(self, result):
  400. """Send a result to the builder for processing
  401. Args:
  402. result: CommandResult object containing the results of the build
  403. Raises:
  404. ValueError if self.test_exception is true (for testing)
  405. """
  406. if self.test_exception:
  407. raise ValueError('test exception')
  408. if self.thread_num != -1:
  409. self.builder.out_queue.put(result)
  410. else:
  411. self.builder.ProcessResult(result)
  412. def RunJob(self, job):
  413. """Run a single job
  414. A job consists of a building a list of commits for a particular board.
  415. Args:
  416. job: Job to build
  417. Returns:
  418. List of Result objects
  419. """
  420. brd = job.board
  421. work_dir = self.builder.GetThreadDir(self.thread_num)
  422. self.toolchain = None
  423. if job.commits:
  424. # Run 'make board_defconfig' on the first commit
  425. do_config = True
  426. commit_upto = 0
  427. force_build = False
  428. for commit_upto in range(0, len(job.commits), job.step):
  429. result, request_config = self.RunCommit(commit_upto, brd,
  430. work_dir, do_config, self.builder.config_only,
  431. force_build or self.builder.force_build,
  432. self.builder.force_build_failures,
  433. work_in_output=job.work_in_output)
  434. failed = result.return_code or result.stderr
  435. did_config = do_config
  436. if failed and not do_config:
  437. # If our incremental build failed, try building again
  438. # with a reconfig.
  439. if self.builder.force_config_on_failure:
  440. result, request_config = self.RunCommit(commit_upto,
  441. brd, work_dir, True, False, True, False,
  442. work_in_output=job.work_in_output)
  443. did_config = True
  444. if not self.builder.force_reconfig:
  445. do_config = request_config
  446. # If we built that commit, then config is done. But if we got
  447. # an warning, reconfig next time to force it to build the same
  448. # files that created warnings this time. Otherwise an
  449. # incremental build may not build the same file, and we will
  450. # think that the warning has gone away.
  451. # We could avoid this by using -Werror everywhere...
  452. # For errors, the problem doesn't happen, since presumably
  453. # the build stopped and didn't generate output, so will retry
  454. # that file next time. So we could detect warnings and deal
  455. # with them specially here. For now, we just reconfigure if
  456. # anything goes work.
  457. # Of course this is substantially slower if there are build
  458. # errors/warnings (e.g. 2-3x slower even if only 10% of builds
  459. # have problems).
  460. if (failed and not result.already_done and not did_config and
  461. self.builder.force_config_on_failure):
  462. # If this build failed, try the next one with a
  463. # reconfigure.
  464. # Sometimes if the board_config.h file changes it can mess
  465. # with dependencies, and we get:
  466. # make: *** No rule to make target `include/autoconf.mk',
  467. # needed by `depend'.
  468. do_config = True
  469. force_build = True
  470. else:
  471. force_build = False
  472. if self.builder.force_config_on_failure:
  473. if failed:
  474. do_config = True
  475. result.commit_upto = commit_upto
  476. if result.return_code < 0:
  477. raise ValueError('Interrupt')
  478. # We have the build results, so output the result
  479. self._WriteResult(result, job.keep_outputs, job.work_in_output)
  480. self._SendResult(result)
  481. else:
  482. # Just build the currently checked-out build
  483. result, request_config = self.RunCommit(None, brd, work_dir, True,
  484. self.builder.config_only, True,
  485. self.builder.force_build_failures,
  486. work_in_output=job.work_in_output)
  487. result.commit_upto = 0
  488. self._WriteResult(result, job.keep_outputs, job.work_in_output)
  489. self._SendResult(result)
  490. def run(self):
  491. """Our thread's run function
  492. This thread picks a job from the queue, runs it, and then goes to the
  493. next job.
  494. """
  495. while True:
  496. job = self.builder.queue.get()
  497. try:
  498. self.RunJob(job)
  499. except Exception as e:
  500. print('Thread exception:', e)
  501. self.builder.thread_exceptions.append(e)
  502. self.builder.queue.task_done()