builder.py 56 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445
  1. # Copyright (c) 2013 The Chromium OS Authors.
  2. #
  3. # Bloat-o-meter code used here Copyright 2004 Matt Mackall <mpm@selenic.com>
  4. #
  5. # See file CREDITS for list of people who contributed to this
  6. # project.
  7. #
  8. # This program is free software; you can redistribute it and/or
  9. # modify it under the terms of the GNU General Public License as
  10. # published by the Free Software Foundation; either version 2 of
  11. # the License, or (at your option) any later version.
  12. #
  13. # This program is distributed in the hope that it will be useful,
  14. # but WITHOUT ANY WARRANTY; without even the implied warranty of
  15. # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  16. # GNU General Public License for more details.
  17. #
  18. # You should have received a copy of the GNU General Public License
  19. # along with this program; if not, write to the Free Software
  20. # Foundation, Inc., 59 Temple Place, Suite 330, Boston,
  21. # MA 02111-1307 USA
  22. #
  23. import collections
  24. import errno
  25. from datetime import datetime, timedelta
  26. import glob
  27. import os
  28. import re
  29. import Queue
  30. import shutil
  31. import string
  32. import sys
  33. import threading
  34. import time
  35. import command
  36. import gitutil
  37. import terminal
  38. import toolchain
  39. """
  40. Theory of Operation
  41. Please see README for user documentation, and you should be familiar with
  42. that before trying to make sense of this.
  43. Buildman works by keeping the machine as busy as possible, building different
  44. commits for different boards on multiple CPUs at once.
  45. The source repo (self.git_dir) contains all the commits to be built. Each
  46. thread works on a single board at a time. It checks out the first commit,
  47. configures it for that board, then builds it. Then it checks out the next
  48. commit and builds it (typically without re-configuring). When it runs out
  49. of commits, it gets another job from the builder and starts again with that
  50. board.
  51. Clearly the builder threads could work either way - they could check out a
  52. commit and then built it for all boards. Using separate directories for each
  53. commit/board pair they could leave their build product around afterwards
  54. also.
  55. The intent behind building a single board for multiple commits, is to make
  56. use of incremental builds. Since each commit is built incrementally from
  57. the previous one, builds are faster. Reconfiguring for a different board
  58. removes all intermediate object files.
  59. Many threads can be working at once, but each has its own working directory.
  60. When a thread finishes a build, it puts the output files into a result
  61. directory.
  62. The base directory used by buildman is normally '../<branch>', i.e.
  63. a directory higher than the source repository and named after the branch
  64. being built.
  65. Within the base directory, we have one subdirectory for each commit. Within
  66. that is one subdirectory for each board. Within that is the build output for
  67. that commit/board combination.
  68. Buildman also create working directories for each thread, in a .bm-work/
  69. subdirectory in the base dir.
  70. As an example, say we are building branch 'us-net' for boards 'sandbox' and
  71. 'seaboard', and say that us-net has two commits. We will have directories
  72. like this:
  73. us-net/ base directory
  74. 01_of_02_g4ed4ebc_net--Add-tftp-speed-/
  75. sandbox/
  76. u-boot.bin
  77. seaboard/
  78. u-boot.bin
  79. 02_of_02_g4ed4ebc_net--Check-tftp-comp/
  80. sandbox/
  81. u-boot.bin
  82. seaboard/
  83. u-boot.bin
  84. .bm-work/
  85. 00/ working directory for thread 0 (contains source checkout)
  86. build/ build output
  87. 01/ working directory for thread 1
  88. build/ build output
  89. ...
  90. u-boot/ source directory
  91. .git/ repository
  92. """
  93. # Possible build outcomes
  94. OUTCOME_OK, OUTCOME_WARNING, OUTCOME_ERROR, OUTCOME_UNKNOWN = range(4)
  95. # Translate a commit subject into a valid filename
  96. trans_valid_chars = string.maketrans("/: ", "---")
  97. def Mkdir(dirname):
  98. """Make a directory if it doesn't already exist.
  99. Args:
  100. dirname: Directory to create
  101. """
  102. try:
  103. os.mkdir(dirname)
  104. except OSError as err:
  105. if err.errno == errno.EEXIST:
  106. pass
  107. else:
  108. raise
  109. class BuilderJob:
  110. """Holds information about a job to be performed by a thread
  111. Members:
  112. board: Board object to build
  113. commits: List of commit options to build.
  114. """
  115. def __init__(self):
  116. self.board = None
  117. self.commits = []
  118. class ResultThread(threading.Thread):
  119. """This thread processes results from builder threads.
  120. It simply passes the results on to the builder. There is only one
  121. result thread, and this helps to serialise the build output.
  122. """
  123. def __init__(self, builder):
  124. """Set up a new result thread
  125. Args:
  126. builder: Builder which will be sent each result
  127. """
  128. threading.Thread.__init__(self)
  129. self.builder = builder
  130. def run(self):
  131. """Called to start up the result thread.
  132. We collect the next result job and pass it on to the build.
  133. """
  134. while True:
  135. result = self.builder.out_queue.get()
  136. self.builder.ProcessResult(result)
  137. self.builder.out_queue.task_done()
  138. class BuilderThread(threading.Thread):
  139. """This thread builds U-Boot for a particular board.
  140. An input queue provides each new job. We run 'make' to build U-Boot
  141. and then pass the results on to the output queue.
  142. Members:
  143. builder: The builder which contains information we might need
  144. thread_num: Our thread number (0-n-1), used to decide on a
  145. temporary directory
  146. """
  147. def __init__(self, builder, thread_num):
  148. """Set up a new builder thread"""
  149. threading.Thread.__init__(self)
  150. self.builder = builder
  151. self.thread_num = thread_num
  152. def Make(self, commit, brd, stage, cwd, *args, **kwargs):
  153. """Run 'make' on a particular commit and board.
  154. The source code will already be checked out, so the 'commit'
  155. argument is only for information.
  156. Args:
  157. commit: Commit object that is being built
  158. brd: Board object that is being built
  159. stage: Stage of the build. Valid stages are:
  160. distclean - can be called to clean source
  161. config - called to configure for a board
  162. build - the main make invocation - it does the build
  163. args: A list of arguments to pass to 'make'
  164. kwargs: A list of keyword arguments to pass to command.RunPipe()
  165. Returns:
  166. CommandResult object
  167. """
  168. return self.builder.do_make(commit, brd, stage, cwd, *args,
  169. **kwargs)
  170. def RunCommit(self, commit_upto, brd, work_dir, do_config, force_build):
  171. """Build a particular commit.
  172. If the build is already done, and we are not forcing a build, we skip
  173. the build and just return the previously-saved results.
  174. Args:
  175. commit_upto: Commit number to build (0...n-1)
  176. brd: Board object to build
  177. work_dir: Directory to which the source will be checked out
  178. do_config: True to run a make <board>_config on the source
  179. force_build: Force a build even if one was previously done
  180. Returns:
  181. tuple containing:
  182. - CommandResult object containing the results of the build
  183. - boolean indicating whether 'make config' is still needed
  184. """
  185. # Create a default result - it will be overwritte by the call to
  186. # self.Make() below, in the event that we do a build.
  187. result = command.CommandResult()
  188. result.return_code = 0
  189. out_dir = os.path.join(work_dir, 'build')
  190. # Check if the job was already completed last time
  191. done_file = self.builder.GetDoneFile(commit_upto, brd.target)
  192. result.already_done = os.path.exists(done_file)
  193. if result.already_done and not force_build:
  194. # Get the return code from that build and use it
  195. with open(done_file, 'r') as fd:
  196. result.return_code = int(fd.readline())
  197. err_file = self.builder.GetErrFile(commit_upto, brd.target)
  198. if os.path.exists(err_file) and os.stat(err_file).st_size:
  199. result.stderr = 'bad'
  200. else:
  201. # We are going to have to build it. First, get a toolchain
  202. if not self.toolchain:
  203. try:
  204. self.toolchain = self.builder.toolchains.Select(brd.arch)
  205. except ValueError as err:
  206. result.return_code = 10
  207. result.stdout = ''
  208. result.stderr = str(err)
  209. # TODO(sjg@chromium.org): This gets swallowed, but needs
  210. # to be reported.
  211. if self.toolchain:
  212. # Checkout the right commit
  213. if commit_upto is not None:
  214. commit = self.builder.commits[commit_upto]
  215. if self.builder.checkout:
  216. git_dir = os.path.join(work_dir, '.git')
  217. gitutil.Checkout(commit.hash, git_dir, work_dir,
  218. force=True)
  219. else:
  220. commit = self.builder.commit # Ick, fix this for BuildCommits()
  221. # Set up the environment and command line
  222. env = self.toolchain.MakeEnvironment()
  223. Mkdir(out_dir)
  224. args = ['O=build', '-s']
  225. if self.builder.num_jobs is not None:
  226. args.extend(['-j', str(self.builder.num_jobs)])
  227. config_args = ['%s_config' % brd.target]
  228. config_out = ''
  229. # If we need to reconfigure, do that now
  230. if do_config:
  231. result = self.Make(commit, brd, 'distclean', work_dir,
  232. 'distclean', *args, env=env)
  233. result = self.Make(commit, brd, 'config', work_dir,
  234. *(args + config_args), env=env)
  235. config_out = result.combined
  236. do_config = False # No need to configure next time
  237. if result.return_code == 0:
  238. result = self.Make(commit, brd, 'build', work_dir, *args,
  239. env=env)
  240. result.stdout = config_out + result.stdout
  241. else:
  242. result.return_code = 1
  243. result.stderr = 'No tool chain for %s\n' % brd.arch
  244. result.already_done = False
  245. result.toolchain = self.toolchain
  246. result.brd = brd
  247. result.commit_upto = commit_upto
  248. result.out_dir = out_dir
  249. return result, do_config
  250. def _WriteResult(self, result, keep_outputs):
  251. """Write a built result to the output directory.
  252. Args:
  253. result: CommandResult object containing result to write
  254. keep_outputs: True to store the output binaries, False
  255. to delete them
  256. """
  257. # Fatal error
  258. if result.return_code < 0:
  259. return
  260. # Aborted?
  261. if result.stderr and 'No child processes' in result.stderr:
  262. return
  263. if result.already_done:
  264. return
  265. # Write the output and stderr
  266. output_dir = self.builder._GetOutputDir(result.commit_upto)
  267. Mkdir(output_dir)
  268. build_dir = self.builder.GetBuildDir(result.commit_upto,
  269. result.brd.target)
  270. Mkdir(build_dir)
  271. outfile = os.path.join(build_dir, 'log')
  272. with open(outfile, 'w') as fd:
  273. if result.stdout:
  274. fd.write(result.stdout)
  275. errfile = self.builder.GetErrFile(result.commit_upto,
  276. result.brd.target)
  277. if result.stderr:
  278. with open(errfile, 'w') as fd:
  279. fd.write(result.stderr)
  280. elif os.path.exists(errfile):
  281. os.remove(errfile)
  282. if result.toolchain:
  283. # Write the build result and toolchain information.
  284. done_file = self.builder.GetDoneFile(result.commit_upto,
  285. result.brd.target)
  286. with open(done_file, 'w') as fd:
  287. fd.write('%s' % result.return_code)
  288. with open(os.path.join(build_dir, 'toolchain'), 'w') as fd:
  289. print >>fd, 'gcc', result.toolchain.gcc
  290. print >>fd, 'path', result.toolchain.path
  291. print >>fd, 'cross', result.toolchain.cross
  292. print >>fd, 'arch', result.toolchain.arch
  293. fd.write('%s' % result.return_code)
  294. with open(os.path.join(build_dir, 'toolchain'), 'w') as fd:
  295. print >>fd, 'gcc', result.toolchain.gcc
  296. print >>fd, 'path', result.toolchain.path
  297. # Write out the image and function size information and an objdump
  298. env = result.toolchain.MakeEnvironment()
  299. lines = []
  300. for fname in ['u-boot', 'spl/u-boot-spl']:
  301. cmd = ['%snm' % self.toolchain.cross, '--size-sort', fname]
  302. nm_result = command.RunPipe([cmd], capture=True,
  303. capture_stderr=True, cwd=result.out_dir,
  304. raise_on_error=False, env=env)
  305. if nm_result.stdout:
  306. nm = self.builder.GetFuncSizesFile(result.commit_upto,
  307. result.brd.target, fname)
  308. with open(nm, 'w') as fd:
  309. print >>fd, nm_result.stdout,
  310. cmd = ['%sobjdump' % self.toolchain.cross, '-h', fname]
  311. dump_result = command.RunPipe([cmd], capture=True,
  312. capture_stderr=True, cwd=result.out_dir,
  313. raise_on_error=False, env=env)
  314. rodata_size = ''
  315. if dump_result.stdout:
  316. objdump = self.builder.GetObjdumpFile(result.commit_upto,
  317. result.brd.target, fname)
  318. with open(objdump, 'w') as fd:
  319. print >>fd, dump_result.stdout,
  320. for line in dump_result.stdout.splitlines():
  321. fields = line.split()
  322. if len(fields) > 5 and fields[1] == '.rodata':
  323. rodata_size = fields[2]
  324. cmd = ['%ssize' % self.toolchain.cross, fname]
  325. size_result = command.RunPipe([cmd], capture=True,
  326. capture_stderr=True, cwd=result.out_dir,
  327. raise_on_error=False, env=env)
  328. if size_result.stdout:
  329. lines.append(size_result.stdout.splitlines()[1] + ' ' +
  330. rodata_size)
  331. # Write out the image sizes file. This is similar to the output
  332. # of binutil's 'size' utility, but it omits the header line and
  333. # adds an additional hex value at the end of each line for the
  334. # rodata size
  335. if len(lines):
  336. sizes = self.builder.GetSizesFile(result.commit_upto,
  337. result.brd.target)
  338. with open(sizes, 'w') as fd:
  339. print >>fd, '\n'.join(lines)
  340. # Now write the actual build output
  341. if keep_outputs:
  342. patterns = ['u-boot', '*.bin', 'u-boot.dtb', '*.map',
  343. 'include/autoconf.mk', 'spl/u-boot-spl',
  344. 'spl/u-boot-spl.bin']
  345. for pattern in patterns:
  346. file_list = glob.glob(os.path.join(result.out_dir, pattern))
  347. for fname in file_list:
  348. shutil.copy(fname, build_dir)
  349. def RunJob(self, job):
  350. """Run a single job
  351. A job consists of a building a list of commits for a particular board.
  352. Args:
  353. job: Job to build
  354. """
  355. brd = job.board
  356. work_dir = self.builder.GetThreadDir(self.thread_num)
  357. self.toolchain = None
  358. if job.commits:
  359. # Run 'make board_config' on the first commit
  360. do_config = True
  361. commit_upto = 0
  362. force_build = False
  363. for commit_upto in range(0, len(job.commits), job.step):
  364. result, request_config = self.RunCommit(commit_upto, brd,
  365. work_dir, do_config,
  366. force_build or self.builder.force_build)
  367. failed = result.return_code or result.stderr
  368. if failed and not do_config:
  369. # If our incremental build failed, try building again
  370. # with a reconfig.
  371. if self.builder.force_config_on_failure:
  372. result, request_config = self.RunCommit(commit_upto,
  373. brd, work_dir, True, True)
  374. do_config = request_config
  375. # If we built that commit, then config is done. But if we got
  376. # an warning, reconfig next time to force it to build the same
  377. # files that created warnings this time. Otherwise an
  378. # incremental build may not build the same file, and we will
  379. # think that the warning has gone away.
  380. # We could avoid this by using -Werror everywhere...
  381. # For errors, the problem doesn't happen, since presumably
  382. # the build stopped and didn't generate output, so will retry
  383. # that file next time. So we could detect warnings and deal
  384. # with them specially here. For now, we just reconfigure if
  385. # anything goes work.
  386. # Of course this is substantially slower if there are build
  387. # errors/warnings (e.g. 2-3x slower even if only 10% of builds
  388. # have problems).
  389. if (failed and not result.already_done and not do_config and
  390. self.builder.force_config_on_failure):
  391. # If this build failed, try the next one with a
  392. # reconfigure.
  393. # Sometimes if the board_config.h file changes it can mess
  394. # with dependencies, and we get:
  395. # make: *** No rule to make target `include/autoconf.mk',
  396. # needed by `depend'.
  397. do_config = True
  398. force_build = True
  399. else:
  400. force_build = False
  401. if self.builder.force_config_on_failure:
  402. if failed:
  403. do_config = True
  404. result.commit_upto = commit_upto
  405. if result.return_code < 0:
  406. raise ValueError('Interrupt')
  407. # We have the build results, so output the result
  408. self._WriteResult(result, job.keep_outputs)
  409. self.builder.out_queue.put(result)
  410. else:
  411. # Just build the currently checked-out build
  412. result = self.RunCommit(None, True)
  413. result.commit_upto = self.builder.upto
  414. self.builder.out_queue.put(result)
  415. def run(self):
  416. """Our thread's run function
  417. This thread picks a job from the queue, runs it, and then goes to the
  418. next job.
  419. """
  420. alive = True
  421. while True:
  422. job = self.builder.queue.get()
  423. try:
  424. if self.builder.active and alive:
  425. self.RunJob(job)
  426. except Exception as err:
  427. alive = False
  428. print err
  429. self.builder.queue.task_done()
  430. class Builder:
  431. """Class for building U-Boot for a particular commit.
  432. Public members: (many should ->private)
  433. active: True if the builder is active and has not been stopped
  434. already_done: Number of builds already completed
  435. base_dir: Base directory to use for builder
  436. checkout: True to check out source, False to skip that step.
  437. This is used for testing.
  438. col: terminal.Color() object
  439. count: Number of commits to build
  440. do_make: Method to call to invoke Make
  441. fail: Number of builds that failed due to error
  442. force_build: Force building even if a build already exists
  443. force_config_on_failure: If a commit fails for a board, disable
  444. incremental building for the next commit we build for that
  445. board, so that we will see all warnings/errors again.
  446. git_dir: Git directory containing source repository
  447. last_line_len: Length of the last line we printed (used for erasing
  448. it with new progress information)
  449. num_jobs: Number of jobs to run at once (passed to make as -j)
  450. num_threads: Number of builder threads to run
  451. out_queue: Queue of results to process
  452. re_make_err: Compiled regular expression for ignore_lines
  453. queue: Queue of jobs to run
  454. threads: List of active threads
  455. toolchains: Toolchains object to use for building
  456. upto: Current commit number we are building (0.count-1)
  457. warned: Number of builds that produced at least one warning
  458. Private members:
  459. _base_board_dict: Last-summarised Dict of boards
  460. _base_err_lines: Last-summarised list of errors
  461. _build_period_us: Time taken for a single build (float object).
  462. _complete_delay: Expected delay until completion (timedelta)
  463. _next_delay_update: Next time we plan to display a progress update
  464. (datatime)
  465. _show_unknown: Show unknown boards (those not built) in summary
  466. _timestamps: List of timestamps for the completion of the last
  467. last _timestamp_count builds. Each is a datetime object.
  468. _timestamp_count: Number of timestamps to keep in our list.
  469. _working_dir: Base working directory containing all threads
  470. """
  471. class Outcome:
  472. """Records a build outcome for a single make invocation
  473. Public Members:
  474. rc: Outcome value (OUTCOME_...)
  475. err_lines: List of error lines or [] if none
  476. sizes: Dictionary of image size information, keyed by filename
  477. - Each value is itself a dictionary containing
  478. values for 'text', 'data' and 'bss', being the integer
  479. size in bytes of each section.
  480. func_sizes: Dictionary keyed by filename - e.g. 'u-boot'. Each
  481. value is itself a dictionary:
  482. key: function name
  483. value: Size of function in bytes
  484. """
  485. def __init__(self, rc, err_lines, sizes, func_sizes):
  486. self.rc = rc
  487. self.err_lines = err_lines
  488. self.sizes = sizes
  489. self.func_sizes = func_sizes
  490. def __init__(self, toolchains, base_dir, git_dir, num_threads, num_jobs,
  491. checkout=True, show_unknown=True, step=1):
  492. """Create a new Builder object
  493. Args:
  494. toolchains: Toolchains object to use for building
  495. base_dir: Base directory to use for builder
  496. git_dir: Git directory containing source repository
  497. num_threads: Number of builder threads to run
  498. num_jobs: Number of jobs to run at once (passed to make as -j)
  499. checkout: True to check out source, False to skip that step.
  500. This is used for testing.
  501. show_unknown: Show unknown boards (those not built) in summary
  502. step: 1 to process every commit, n to process every nth commit
  503. """
  504. self.toolchains = toolchains
  505. self.base_dir = base_dir
  506. self._working_dir = os.path.join(base_dir, '.bm-work')
  507. self.threads = []
  508. self.active = True
  509. self.do_make = self.Make
  510. self.checkout = checkout
  511. self.num_threads = num_threads
  512. self.num_jobs = num_jobs
  513. self.already_done = 0
  514. self.force_build = False
  515. self.git_dir = git_dir
  516. self._show_unknown = show_unknown
  517. self._timestamp_count = 10
  518. self._build_period_us = None
  519. self._complete_delay = None
  520. self._next_delay_update = datetime.now()
  521. self.force_config_on_failure = True
  522. self._step = step
  523. self.col = terminal.Color()
  524. self.queue = Queue.Queue()
  525. self.out_queue = Queue.Queue()
  526. for i in range(self.num_threads):
  527. t = BuilderThread(self, i)
  528. t.setDaemon(True)
  529. t.start()
  530. self.threads.append(t)
  531. self.last_line_len = 0
  532. t = ResultThread(self)
  533. t.setDaemon(True)
  534. t.start()
  535. self.threads.append(t)
  536. ignore_lines = ['(make.*Waiting for unfinished)', '(Segmentation fault)']
  537. self.re_make_err = re.compile('|'.join(ignore_lines))
  538. def __del__(self):
  539. """Get rid of all threads created by the builder"""
  540. for t in self.threads:
  541. del t
  542. def _AddTimestamp(self):
  543. """Add a new timestamp to the list and record the build period.
  544. The build period is the length of time taken to perform a single
  545. build (one board, one commit).
  546. """
  547. now = datetime.now()
  548. self._timestamps.append(now)
  549. count = len(self._timestamps)
  550. delta = self._timestamps[-1] - self._timestamps[0]
  551. seconds = delta.total_seconds()
  552. # If we have enough data, estimate build period (time taken for a
  553. # single build) and therefore completion time.
  554. if count > 1 and self._next_delay_update < now:
  555. self._next_delay_update = now + timedelta(seconds=2)
  556. if seconds > 0:
  557. self._build_period = float(seconds) / count
  558. todo = self.count - self.upto
  559. self._complete_delay = timedelta(microseconds=
  560. self._build_period * todo * 1000000)
  561. # Round it
  562. self._complete_delay -= timedelta(
  563. microseconds=self._complete_delay.microseconds)
  564. if seconds > 60:
  565. self._timestamps.popleft()
  566. count -= 1
  567. def ClearLine(self, length):
  568. """Clear any characters on the current line
  569. Make way for a new line of length 'length', by outputting enough
  570. spaces to clear out the old line. Then remember the new length for
  571. next time.
  572. Args:
  573. length: Length of new line, in characters
  574. """
  575. if length < self.last_line_len:
  576. print ' ' * (self.last_line_len - length),
  577. print '\r',
  578. self.last_line_len = length
  579. sys.stdout.flush()
  580. def SelectCommit(self, commit, checkout=True):
  581. """Checkout the selected commit for this build
  582. """
  583. self.commit = commit
  584. if checkout and self.checkout:
  585. gitutil.Checkout(commit.hash)
  586. def Make(self, commit, brd, stage, cwd, *args, **kwargs):
  587. """Run make
  588. Args:
  589. commit: Commit object that is being built
  590. brd: Board object that is being built
  591. stage: Stage that we are at (distclean, config, build)
  592. cwd: Directory where make should be run
  593. args: Arguments to pass to make
  594. kwargs: Arguments to pass to command.RunPipe()
  595. """
  596. cmd = ['make'] + list(args)
  597. result = command.RunPipe([cmd], capture=True, capture_stderr=True,
  598. cwd=cwd, raise_on_error=False, **kwargs)
  599. return result
  600. def ProcessResult(self, result):
  601. """Process the result of a build, showing progress information
  602. Args:
  603. result: A CommandResult object
  604. """
  605. col = terminal.Color()
  606. if result:
  607. target = result.brd.target
  608. if result.return_code < 0:
  609. self.active = False
  610. command.StopAll()
  611. return
  612. self.upto += 1
  613. if result.return_code != 0:
  614. self.fail += 1
  615. elif result.stderr:
  616. self.warned += 1
  617. if result.already_done:
  618. self.already_done += 1
  619. else:
  620. target = '(starting)'
  621. # Display separate counts for ok, warned and fail
  622. ok = self.upto - self.warned - self.fail
  623. line = '\r' + self.col.Color(self.col.GREEN, '%5d' % ok)
  624. line += self.col.Color(self.col.YELLOW, '%5d' % self.warned)
  625. line += self.col.Color(self.col.RED, '%5d' % self.fail)
  626. name = ' /%-5d ' % self.count
  627. # Add our current completion time estimate
  628. self._AddTimestamp()
  629. if self._complete_delay:
  630. name += '%s : ' % self._complete_delay
  631. # When building all boards for a commit, we can print a commit
  632. # progress message.
  633. if result and result.commit_upto is None:
  634. name += 'commit %2d/%-3d' % (self.commit_upto + 1,
  635. self.commit_count)
  636. name += target
  637. print line + name,
  638. length = 13 + len(name)
  639. self.ClearLine(length)
  640. def _GetOutputDir(self, commit_upto):
  641. """Get the name of the output directory for a commit number
  642. The output directory is typically .../<branch>/<commit>.
  643. Args:
  644. commit_upto: Commit number to use (0..self.count-1)
  645. """
  646. commit = self.commits[commit_upto]
  647. subject = commit.subject.translate(trans_valid_chars)
  648. commit_dir = ('%02d_of_%02d_g%s_%s' % (commit_upto + 1,
  649. self.commit_count, commit.hash, subject[:20]))
  650. output_dir = os.path.join(self.base_dir, commit_dir)
  651. return output_dir
  652. def GetBuildDir(self, commit_upto, target):
  653. """Get the name of the build directory for a commit number
  654. The build directory is typically .../<branch>/<commit>/<target>.
  655. Args:
  656. commit_upto: Commit number to use (0..self.count-1)
  657. target: Target name
  658. """
  659. output_dir = self._GetOutputDir(commit_upto)
  660. return os.path.join(output_dir, target)
  661. def GetDoneFile(self, commit_upto, target):
  662. """Get the name of the done file for a commit number
  663. Args:
  664. commit_upto: Commit number to use (0..self.count-1)
  665. target: Target name
  666. """
  667. return os.path.join(self.GetBuildDir(commit_upto, target), 'done')
  668. def GetSizesFile(self, commit_upto, target):
  669. """Get the name of the sizes file for a commit number
  670. Args:
  671. commit_upto: Commit number to use (0..self.count-1)
  672. target: Target name
  673. """
  674. return os.path.join(self.GetBuildDir(commit_upto, target), 'sizes')
  675. def GetFuncSizesFile(self, commit_upto, target, elf_fname):
  676. """Get the name of the funcsizes file for a commit number and ELF file
  677. Args:
  678. commit_upto: Commit number to use (0..self.count-1)
  679. target: Target name
  680. elf_fname: Filename of elf image
  681. """
  682. return os.path.join(self.GetBuildDir(commit_upto, target),
  683. '%s.sizes' % elf_fname.replace('/', '-'))
  684. def GetObjdumpFile(self, commit_upto, target, elf_fname):
  685. """Get the name of the objdump file for a commit number and ELF file
  686. Args:
  687. commit_upto: Commit number to use (0..self.count-1)
  688. target: Target name
  689. elf_fname: Filename of elf image
  690. """
  691. return os.path.join(self.GetBuildDir(commit_upto, target),
  692. '%s.objdump' % elf_fname.replace('/', '-'))
  693. def GetErrFile(self, commit_upto, target):
  694. """Get the name of the err file for a commit number
  695. Args:
  696. commit_upto: Commit number to use (0..self.count-1)
  697. target: Target name
  698. """
  699. output_dir = self.GetBuildDir(commit_upto, target)
  700. return os.path.join(output_dir, 'err')
  701. def FilterErrors(self, lines):
  702. """Filter out errors in which we have no interest
  703. We should probably use map().
  704. Args:
  705. lines: List of error lines, each a string
  706. Returns:
  707. New list with only interesting lines included
  708. """
  709. out_lines = []
  710. for line in lines:
  711. if not self.re_make_err.search(line):
  712. out_lines.append(line)
  713. return out_lines
  714. def ReadFuncSizes(self, fname, fd):
  715. """Read function sizes from the output of 'nm'
  716. Args:
  717. fd: File containing data to read
  718. fname: Filename we are reading from (just for errors)
  719. Returns:
  720. Dictionary containing size of each function in bytes, indexed by
  721. function name.
  722. """
  723. sym = {}
  724. for line in fd.readlines():
  725. try:
  726. size, type, name = line[:-1].split()
  727. except:
  728. print "Invalid line in file '%s': '%s'" % (fname, line[:-1])
  729. continue
  730. if type in 'tTdDbB':
  731. # function names begin with '.' on 64-bit powerpc
  732. if '.' in name[1:]:
  733. name = 'static.' + name.split('.')[0]
  734. sym[name] = sym.get(name, 0) + int(size, 16)
  735. return sym
  736. def GetBuildOutcome(self, commit_upto, target, read_func_sizes):
  737. """Work out the outcome of a build.
  738. Args:
  739. commit_upto: Commit number to check (0..n-1)
  740. target: Target board to check
  741. read_func_sizes: True to read function size information
  742. Returns:
  743. Outcome object
  744. """
  745. done_file = self.GetDoneFile(commit_upto, target)
  746. sizes_file = self.GetSizesFile(commit_upto, target)
  747. sizes = {}
  748. func_sizes = {}
  749. if os.path.exists(done_file):
  750. with open(done_file, 'r') as fd:
  751. return_code = int(fd.readline())
  752. err_lines = []
  753. err_file = self.GetErrFile(commit_upto, target)
  754. if os.path.exists(err_file):
  755. with open(err_file, 'r') as fd:
  756. err_lines = self.FilterErrors(fd.readlines())
  757. # Decide whether the build was ok, failed or created warnings
  758. if return_code:
  759. rc = OUTCOME_ERROR
  760. elif len(err_lines):
  761. rc = OUTCOME_WARNING
  762. else:
  763. rc = OUTCOME_OK
  764. # Convert size information to our simple format
  765. if os.path.exists(sizes_file):
  766. with open(sizes_file, 'r') as fd:
  767. for line in fd.readlines():
  768. values = line.split()
  769. rodata = 0
  770. if len(values) > 6:
  771. rodata = int(values[6], 16)
  772. size_dict = {
  773. 'all' : int(values[0]) + int(values[1]) +
  774. int(values[2]),
  775. 'text' : int(values[0]) - rodata,
  776. 'data' : int(values[1]),
  777. 'bss' : int(values[2]),
  778. 'rodata' : rodata,
  779. }
  780. sizes[values[5]] = size_dict
  781. if read_func_sizes:
  782. pattern = self.GetFuncSizesFile(commit_upto, target, '*')
  783. for fname in glob.glob(pattern):
  784. with open(fname, 'r') as fd:
  785. dict_name = os.path.basename(fname).replace('.sizes',
  786. '')
  787. func_sizes[dict_name] = self.ReadFuncSizes(fname, fd)
  788. return Builder.Outcome(rc, err_lines, sizes, func_sizes)
  789. return Builder.Outcome(OUTCOME_UNKNOWN, [], {}, {})
  790. def GetResultSummary(self, boards_selected, commit_upto, read_func_sizes):
  791. """Calculate a summary of the results of building a commit.
  792. Args:
  793. board_selected: Dict containing boards to summarise
  794. commit_upto: Commit number to summarize (0..self.count-1)
  795. read_func_sizes: True to read function size information
  796. Returns:
  797. Tuple:
  798. Dict containing boards which passed building this commit.
  799. keyed by board.target
  800. List containing a summary of error/warning lines
  801. """
  802. board_dict = {}
  803. err_lines_summary = []
  804. for board in boards_selected.itervalues():
  805. outcome = self.GetBuildOutcome(commit_upto, board.target,
  806. read_func_sizes)
  807. board_dict[board.target] = outcome
  808. for err in outcome.err_lines:
  809. if err and not err.rstrip() in err_lines_summary:
  810. err_lines_summary.append(err.rstrip())
  811. return board_dict, err_lines_summary
  812. def AddOutcome(self, board_dict, arch_list, changes, char, color):
  813. """Add an output to our list of outcomes for each architecture
  814. This simple function adds failing boards (changes) to the
  815. relevant architecture string, so we can print the results out
  816. sorted by architecture.
  817. Args:
  818. board_dict: Dict containing all boards
  819. arch_list: Dict keyed by arch name. Value is a string containing
  820. a list of board names which failed for that arch.
  821. changes: List of boards to add to arch_list
  822. color: terminal.Colour object
  823. """
  824. done_arch = {}
  825. for target in changes:
  826. if target in board_dict:
  827. arch = board_dict[target].arch
  828. else:
  829. arch = 'unknown'
  830. str = self.col.Color(color, ' ' + target)
  831. if not arch in done_arch:
  832. str = self.col.Color(color, char) + ' ' + str
  833. done_arch[arch] = True
  834. if not arch in arch_list:
  835. arch_list[arch] = str
  836. else:
  837. arch_list[arch] += str
  838. def ColourNum(self, num):
  839. color = self.col.RED if num > 0 else self.col.GREEN
  840. if num == 0:
  841. return '0'
  842. return self.col.Color(color, str(num))
  843. def ResetResultSummary(self, board_selected):
  844. """Reset the results summary ready for use.
  845. Set up the base board list to be all those selected, and set the
  846. error lines to empty.
  847. Following this, calls to PrintResultSummary() will use this
  848. information to work out what has changed.
  849. Args:
  850. board_selected: Dict containing boards to summarise, keyed by
  851. board.target
  852. """
  853. self._base_board_dict = {}
  854. for board in board_selected:
  855. self._base_board_dict[board] = Builder.Outcome(0, [], [], {})
  856. self._base_err_lines = []
  857. def PrintFuncSizeDetail(self, fname, old, new):
  858. grow, shrink, add, remove, up, down = 0, 0, 0, 0, 0, 0
  859. delta, common = [], {}
  860. for a in old:
  861. if a in new:
  862. common[a] = 1
  863. for name in old:
  864. if name not in common:
  865. remove += 1
  866. down += old[name]
  867. delta.append([-old[name], name])
  868. for name in new:
  869. if name not in common:
  870. add += 1
  871. up += new[name]
  872. delta.append([new[name], name])
  873. for name in common:
  874. diff = new.get(name, 0) - old.get(name, 0)
  875. if diff > 0:
  876. grow, up = grow + 1, up + diff
  877. elif diff < 0:
  878. shrink, down = shrink + 1, down - diff
  879. delta.append([diff, name])
  880. delta.sort()
  881. delta.reverse()
  882. args = [add, -remove, grow, -shrink, up, -down, up - down]
  883. if max(args) == 0:
  884. return
  885. args = [self.ColourNum(x) for x in args]
  886. indent = ' ' * 15
  887. print ('%s%s: add: %s/%s, grow: %s/%s bytes: %s/%s (%s)' %
  888. tuple([indent, self.col.Color(self.col.YELLOW, fname)] + args))
  889. print '%s %-38s %7s %7s %+7s' % (indent, 'function', 'old', 'new',
  890. 'delta')
  891. for diff, name in delta:
  892. if diff:
  893. color = self.col.RED if diff > 0 else self.col.GREEN
  894. msg = '%s %-38s %7s %7s %+7d' % (indent, name,
  895. old.get(name, '-'), new.get(name,'-'), diff)
  896. print self.col.Color(color, msg)
  897. def PrintSizeDetail(self, target_list, show_bloat):
  898. """Show details size information for each board
  899. Args:
  900. target_list: List of targets, each a dict containing:
  901. 'target': Target name
  902. 'total_diff': Total difference in bytes across all areas
  903. <part_name>: Difference for that part
  904. show_bloat: Show detail for each function
  905. """
  906. targets_by_diff = sorted(target_list, reverse=True,
  907. key=lambda x: x['_total_diff'])
  908. for result in targets_by_diff:
  909. printed_target = False
  910. for name in sorted(result):
  911. diff = result[name]
  912. if name.startswith('_'):
  913. continue
  914. if diff != 0:
  915. color = self.col.RED if diff > 0 else self.col.GREEN
  916. msg = ' %s %+d' % (name, diff)
  917. if not printed_target:
  918. print '%10s %-15s:' % ('', result['_target']),
  919. printed_target = True
  920. print self.col.Color(color, msg),
  921. if printed_target:
  922. print
  923. if show_bloat:
  924. target = result['_target']
  925. outcome = result['_outcome']
  926. base_outcome = self._base_board_dict[target]
  927. for fname in outcome.func_sizes:
  928. self.PrintFuncSizeDetail(fname,
  929. base_outcome.func_sizes[fname],
  930. outcome.func_sizes[fname])
  931. def PrintSizeSummary(self, board_selected, board_dict, show_detail,
  932. show_bloat):
  933. """Print a summary of image sizes broken down by section.
  934. The summary takes the form of one line per architecture. The
  935. line contains deltas for each of the sections (+ means the section
  936. got bigger, - means smaller). The nunmbers are the average number
  937. of bytes that a board in this section increased by.
  938. For example:
  939. powerpc: (622 boards) text -0.0
  940. arm: (285 boards) text -0.0
  941. nds32: (3 boards) text -8.0
  942. Args:
  943. board_selected: Dict containing boards to summarise, keyed by
  944. board.target
  945. board_dict: Dict containing boards for which we built this
  946. commit, keyed by board.target. The value is an Outcome object.
  947. show_detail: Show detail for each board
  948. show_bloat: Show detail for each function
  949. """
  950. arch_list = {}
  951. arch_count = {}
  952. # Calculate changes in size for different image parts
  953. # The previous sizes are in Board.sizes, for each board
  954. for target in board_dict:
  955. if target not in board_selected:
  956. continue
  957. base_sizes = self._base_board_dict[target].sizes
  958. outcome = board_dict[target]
  959. sizes = outcome.sizes
  960. # Loop through the list of images, creating a dict of size
  961. # changes for each image/part. We end up with something like
  962. # {'target' : 'snapper9g45, 'data' : 5, 'u-boot-spl:text' : -4}
  963. # which means that U-Boot data increased by 5 bytes and SPL
  964. # text decreased by 4.
  965. err = {'_target' : target}
  966. for image in sizes:
  967. if image in base_sizes:
  968. base_image = base_sizes[image]
  969. # Loop through the text, data, bss parts
  970. for part in sorted(sizes[image]):
  971. diff = sizes[image][part] - base_image[part]
  972. col = None
  973. if diff:
  974. if image == 'u-boot':
  975. name = part
  976. else:
  977. name = image + ':' + part
  978. err[name] = diff
  979. arch = board_selected[target].arch
  980. if not arch in arch_count:
  981. arch_count[arch] = 1
  982. else:
  983. arch_count[arch] += 1
  984. if not sizes:
  985. pass # Only add to our list when we have some stats
  986. elif not arch in arch_list:
  987. arch_list[arch] = [err]
  988. else:
  989. arch_list[arch].append(err)
  990. # We now have a list of image size changes sorted by arch
  991. # Print out a summary of these
  992. for arch, target_list in arch_list.iteritems():
  993. # Get total difference for each type
  994. totals = {}
  995. for result in target_list:
  996. total = 0
  997. for name, diff in result.iteritems():
  998. if name.startswith('_'):
  999. continue
  1000. total += diff
  1001. if name in totals:
  1002. totals[name] += diff
  1003. else:
  1004. totals[name] = diff
  1005. result['_total_diff'] = total
  1006. result['_outcome'] = board_dict[result['_target']]
  1007. count = len(target_list)
  1008. printed_arch = False
  1009. for name in sorted(totals):
  1010. diff = totals[name]
  1011. if diff:
  1012. # Display the average difference in this name for this
  1013. # architecture
  1014. avg_diff = float(diff) / count
  1015. color = self.col.RED if avg_diff > 0 else self.col.GREEN
  1016. msg = ' %s %+1.1f' % (name, avg_diff)
  1017. if not printed_arch:
  1018. print '%10s: (for %d/%d boards)' % (arch, count,
  1019. arch_count[arch]),
  1020. printed_arch = True
  1021. print self.col.Color(color, msg),
  1022. if printed_arch:
  1023. print
  1024. if show_detail:
  1025. self.PrintSizeDetail(target_list, show_bloat)
  1026. def PrintResultSummary(self, board_selected, board_dict, err_lines,
  1027. show_sizes, show_detail, show_bloat):
  1028. """Compare results with the base results and display delta.
  1029. Only boards mentioned in board_selected will be considered. This
  1030. function is intended to be called repeatedly with the results of
  1031. each commit. It therefore shows a 'diff' between what it saw in
  1032. the last call and what it sees now.
  1033. Args:
  1034. board_selected: Dict containing boards to summarise, keyed by
  1035. board.target
  1036. board_dict: Dict containing boards for which we built this
  1037. commit, keyed by board.target. The value is an Outcome object.
  1038. err_lines: A list of errors for this commit, or [] if there is
  1039. none, or we don't want to print errors
  1040. show_sizes: Show image size deltas
  1041. show_detail: Show detail for each board
  1042. show_bloat: Show detail for each function
  1043. """
  1044. better = [] # List of boards fixed since last commit
  1045. worse = [] # List of new broken boards since last commit
  1046. new = [] # List of boards that didn't exist last time
  1047. unknown = [] # List of boards that were not built
  1048. for target in board_dict:
  1049. if target not in board_selected:
  1050. continue
  1051. # If the board was built last time, add its outcome to a list
  1052. if target in self._base_board_dict:
  1053. base_outcome = self._base_board_dict[target].rc
  1054. outcome = board_dict[target]
  1055. if outcome.rc == OUTCOME_UNKNOWN:
  1056. unknown.append(target)
  1057. elif outcome.rc < base_outcome:
  1058. better.append(target)
  1059. elif outcome.rc > base_outcome:
  1060. worse.append(target)
  1061. else:
  1062. new.append(target)
  1063. # Get a list of errors that have appeared, and disappeared
  1064. better_err = []
  1065. worse_err = []
  1066. for line in err_lines:
  1067. if line not in self._base_err_lines:
  1068. worse_err.append('+' + line)
  1069. for line in self._base_err_lines:
  1070. if line not in err_lines:
  1071. better_err.append('-' + line)
  1072. # Display results by arch
  1073. if better or worse or unknown or new or worse_err or better_err:
  1074. arch_list = {}
  1075. self.AddOutcome(board_selected, arch_list, better, '',
  1076. self.col.GREEN)
  1077. self.AddOutcome(board_selected, arch_list, worse, '+',
  1078. self.col.RED)
  1079. self.AddOutcome(board_selected, arch_list, new, '*', self.col.BLUE)
  1080. if self._show_unknown:
  1081. self.AddOutcome(board_selected, arch_list, unknown, '?',
  1082. self.col.MAGENTA)
  1083. for arch, target_list in arch_list.iteritems():
  1084. print '%10s: %s' % (arch, target_list)
  1085. if better_err:
  1086. print self.col.Color(self.col.GREEN, '\n'.join(better_err))
  1087. if worse_err:
  1088. print self.col.Color(self.col.RED, '\n'.join(worse_err))
  1089. if show_sizes:
  1090. self.PrintSizeSummary(board_selected, board_dict, show_detail,
  1091. show_bloat)
  1092. # Save our updated information for the next call to this function
  1093. self._base_board_dict = board_dict
  1094. self._base_err_lines = err_lines
  1095. # Get a list of boards that did not get built, if needed
  1096. not_built = []
  1097. for board in board_selected:
  1098. if not board in board_dict:
  1099. not_built.append(board)
  1100. if not_built:
  1101. print "Boards not built (%d): %s" % (len(not_built),
  1102. ', '.join(not_built))
  1103. def ShowSummary(self, commits, board_selected, show_errors, show_sizes,
  1104. show_detail, show_bloat):
  1105. """Show a build summary for U-Boot for a given board list.
  1106. Reset the result summary, then repeatedly call GetResultSummary on
  1107. each commit's results, then display the differences we see.
  1108. Args:
  1109. commit: Commit objects to summarise
  1110. board_selected: Dict containing boards to summarise
  1111. show_errors: Show errors that occured
  1112. show_sizes: Show size deltas
  1113. show_detail: Show detail for each board
  1114. show_bloat: Show detail for each function
  1115. """
  1116. self.commit_count = len(commits)
  1117. self.commits = commits
  1118. self.ResetResultSummary(board_selected)
  1119. for commit_upto in range(0, self.commit_count, self._step):
  1120. board_dict, err_lines = self.GetResultSummary(board_selected,
  1121. commit_upto, read_func_sizes=show_bloat)
  1122. msg = '%02d: %s' % (commit_upto + 1, commits[commit_upto].subject)
  1123. print self.col.Color(self.col.BLUE, msg)
  1124. self.PrintResultSummary(board_selected, board_dict,
  1125. err_lines if show_errors else [], show_sizes, show_detail,
  1126. show_bloat)
  1127. def SetupBuild(self, board_selected, commits):
  1128. """Set up ready to start a build.
  1129. Args:
  1130. board_selected: Selected boards to build
  1131. commits: Selected commits to build
  1132. """
  1133. # First work out how many commits we will build
  1134. count = (len(commits) + self._step - 1) / self._step
  1135. self.count = len(board_selected) * count
  1136. self.upto = self.warned = self.fail = 0
  1137. self._timestamps = collections.deque()
  1138. def BuildBoardsForCommit(self, board_selected, keep_outputs):
  1139. """Build all boards for a single commit"""
  1140. self.SetupBuild(board_selected)
  1141. self.count = len(board_selected)
  1142. for brd in board_selected.itervalues():
  1143. job = BuilderJob()
  1144. job.board = brd
  1145. job.commits = None
  1146. job.keep_outputs = keep_outputs
  1147. self.queue.put(brd)
  1148. self.queue.join()
  1149. self.out_queue.join()
  1150. print
  1151. self.ClearLine(0)
  1152. def BuildCommits(self, commits, board_selected, show_errors, keep_outputs):
  1153. """Build all boards for all commits (non-incremental)"""
  1154. self.commit_count = len(commits)
  1155. self.ResetResultSummary(board_selected)
  1156. for self.commit_upto in range(self.commit_count):
  1157. self.SelectCommit(commits[self.commit_upto])
  1158. self.SelectOutputDir()
  1159. Mkdir(self.output_dir)
  1160. self.BuildBoardsForCommit(board_selected, keep_outputs)
  1161. board_dict, err_lines = self.GetResultSummary()
  1162. self.PrintResultSummary(board_selected, board_dict,
  1163. err_lines if show_errors else [])
  1164. if self.already_done:
  1165. print '%d builds already done' % self.already_done
  1166. def GetThreadDir(self, thread_num):
  1167. """Get the directory path to the working dir for a thread.
  1168. Args:
  1169. thread_num: Number of thread to check.
  1170. """
  1171. return os.path.join(self._working_dir, '%02d' % thread_num)
  1172. def _PrepareThread(self, thread_num):
  1173. """Prepare the working directory for a thread.
  1174. This clones or fetches the repo into the thread's work directory.
  1175. Args:
  1176. thread_num: Thread number (0, 1, ...)
  1177. """
  1178. thread_dir = self.GetThreadDir(thread_num)
  1179. Mkdir(thread_dir)
  1180. git_dir = os.path.join(thread_dir, '.git')
  1181. # Clone the repo if it doesn't already exist
  1182. # TODO(sjg@chromium): Perhaps some git hackery to symlink instead, so
  1183. # we have a private index but uses the origin repo's contents?
  1184. if self.git_dir:
  1185. src_dir = os.path.abspath(self.git_dir)
  1186. if os.path.exists(git_dir):
  1187. gitutil.Fetch(git_dir, thread_dir)
  1188. else:
  1189. print 'Cloning repo for thread %d' % thread_num
  1190. gitutil.Clone(src_dir, thread_dir)
  1191. def _PrepareWorkingSpace(self, max_threads):
  1192. """Prepare the working directory for use.
  1193. Set up the git repo for each thread.
  1194. Args:
  1195. max_threads: Maximum number of threads we expect to need.
  1196. """
  1197. Mkdir(self._working_dir)
  1198. for thread in range(max_threads):
  1199. self._PrepareThread(thread)
  1200. def _PrepareOutputSpace(self):
  1201. """Get the output directories ready to receive files.
  1202. We delete any output directories which look like ones we need to
  1203. create. Having left over directories is confusing when the user wants
  1204. to check the output manually.
  1205. """
  1206. dir_list = []
  1207. for commit_upto in range(self.commit_count):
  1208. dir_list.append(self._GetOutputDir(commit_upto))
  1209. for dirname in glob.glob(os.path.join(self.base_dir, '*')):
  1210. if dirname not in dir_list:
  1211. shutil.rmtree(dirname)
  1212. def BuildBoards(self, commits, board_selected, show_errors, keep_outputs):
  1213. """Build all commits for a list of boards
  1214. Args:
  1215. commits: List of commits to be build, each a Commit object
  1216. boards_selected: Dict of selected boards, key is target name,
  1217. value is Board object
  1218. show_errors: True to show summarised error/warning info
  1219. keep_outputs: True to save build output files
  1220. """
  1221. self.commit_count = len(commits)
  1222. self.commits = commits
  1223. self.ResetResultSummary(board_selected)
  1224. Mkdir(self.base_dir)
  1225. self._PrepareWorkingSpace(min(self.num_threads, len(board_selected)))
  1226. self._PrepareOutputSpace()
  1227. self.SetupBuild(board_selected, commits)
  1228. self.ProcessResult(None)
  1229. # Create jobs to build all commits for each board
  1230. for brd in board_selected.itervalues():
  1231. job = BuilderJob()
  1232. job.board = brd
  1233. job.commits = commits
  1234. job.keep_outputs = keep_outputs
  1235. job.step = self._step
  1236. self.queue.put(job)
  1237. # Wait until all jobs are started
  1238. self.queue.join()
  1239. # Wait until we have processed all output
  1240. self.out_queue.join()
  1241. print
  1242. self.ClearLine(0)