control.py 13 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349
  1. # SPDX-License-Identifier: GPL-2.0+
  2. # Copyright (c) 2013 The Chromium OS Authors.
  3. #
  4. import multiprocessing
  5. import os
  6. import shutil
  7. import sys
  8. import board
  9. import bsettings
  10. from builder import Builder
  11. import gitutil
  12. import patchstream
  13. import terminal
  14. from terminal import Print
  15. import toolchain
  16. import command
  17. import subprocess
  18. def GetPlural(count):
  19. """Returns a plural 's' if count is not 1"""
  20. return 's' if count != 1 else ''
  21. def GetActionSummary(is_summary, commits, selected, options):
  22. """Return a string summarising the intended action.
  23. Returns:
  24. Summary string.
  25. """
  26. if commits:
  27. count = len(commits)
  28. count = (count + options.step - 1) // options.step
  29. commit_str = '%d commit%s' % (count, GetPlural(count))
  30. else:
  31. commit_str = 'current source'
  32. str = '%s %s for %d boards' % (
  33. 'Summary of' if is_summary else 'Building', commit_str,
  34. len(selected))
  35. str += ' (%d thread%s, %d job%s per thread)' % (options.threads,
  36. GetPlural(options.threads), options.jobs, GetPlural(options.jobs))
  37. return str
  38. def ShowActions(series, why_selected, boards_selected, builder, options,
  39. board_warnings):
  40. """Display a list of actions that we would take, if not a dry run.
  41. Args:
  42. series: Series object
  43. why_selected: Dictionary where each key is a buildman argument
  44. provided by the user, and the value is the list of boards
  45. brought in by that argument. For example, 'arm' might bring
  46. in 400 boards, so in this case the key would be 'arm' and
  47. the value would be a list of board names.
  48. boards_selected: Dict of selected boards, key is target name,
  49. value is Board object
  50. builder: The builder that will be used to build the commits
  51. options: Command line options object
  52. board_warnings: List of warnings obtained from board selected
  53. """
  54. col = terminal.Color()
  55. print('Dry run, so not doing much. But I would do this:')
  56. print()
  57. if series:
  58. commits = series.commits
  59. else:
  60. commits = None
  61. print(GetActionSummary(False, commits, boards_selected,
  62. options))
  63. print('Build directory: %s' % builder.base_dir)
  64. if commits:
  65. for upto in range(0, len(series.commits), options.step):
  66. commit = series.commits[upto]
  67. print(' ', col.Color(col.YELLOW, commit.hash[:8], bright=False), end=' ')
  68. print(commit.subject)
  69. print()
  70. for arg in why_selected:
  71. if arg != 'all':
  72. print(arg, ': %d boards' % len(why_selected[arg]))
  73. if options.verbose:
  74. print(' %s' % ' '.join(why_selected[arg]))
  75. print(('Total boards to build for each commit: %d\n' %
  76. len(why_selected['all'])))
  77. if board_warnings:
  78. for warning in board_warnings:
  79. print(col.Color(col.YELLOW, warning))
  80. def CheckOutputDir(output_dir):
  81. """Make sure that the output directory is not within the current directory
  82. If we try to use an output directory which is within the current directory
  83. (which is assumed to hold the U-Boot source) we may end up deleting the
  84. U-Boot source code. Detect this and print an error in this case.
  85. Args:
  86. output_dir: Output directory path to check
  87. """
  88. path = os.path.realpath(output_dir)
  89. cwd_path = os.path.realpath('.')
  90. while True:
  91. if os.path.realpath(path) == cwd_path:
  92. Print("Cannot use output directory '%s' since it is within the current directory '%s'" %
  93. (path, cwd_path))
  94. sys.exit(1)
  95. parent = os.path.dirname(path)
  96. if parent == path:
  97. break
  98. path = parent
  99. def DoBuildman(options, args, toolchains=None, make_func=None, boards=None,
  100. clean_dir=False):
  101. """The main control code for buildman
  102. Args:
  103. options: Command line options object
  104. args: Command line arguments (list of strings)
  105. toolchains: Toolchains to use - this should be a Toolchains()
  106. object. If None, then it will be created and scanned
  107. make_func: Make function to use for the builder. This is called
  108. to execute 'make'. If this is None, the normal function
  109. will be used, which calls the 'make' tool with suitable
  110. arguments. This setting is useful for tests.
  111. board: Boards() object to use, containing a list of available
  112. boards. If this is None it will be created and scanned.
  113. """
  114. global builder
  115. if options.full_help:
  116. pager = os.getenv('PAGER')
  117. if not pager:
  118. pager = 'more'
  119. fname = os.path.join(os.path.dirname(os.path.realpath(sys.argv[0])),
  120. 'README')
  121. command.Run(pager, fname)
  122. return 0
  123. gitutil.Setup()
  124. col = terminal.Color()
  125. options.git_dir = os.path.join(options.git, '.git')
  126. no_toolchains = toolchains is None
  127. if no_toolchains:
  128. toolchains = toolchain.Toolchains(options.override_toolchain)
  129. if options.fetch_arch:
  130. if options.fetch_arch == 'list':
  131. sorted_list = toolchains.ListArchs()
  132. print(col.Color(col.BLUE, 'Available architectures: %s\n' %
  133. ' '.join(sorted_list)))
  134. return 0
  135. else:
  136. fetch_arch = options.fetch_arch
  137. if fetch_arch == 'all':
  138. fetch_arch = ','.join(toolchains.ListArchs())
  139. print(col.Color(col.CYAN, '\nDownloading toolchains: %s' %
  140. fetch_arch))
  141. for arch in fetch_arch.split(','):
  142. print()
  143. ret = toolchains.FetchAndInstall(arch)
  144. if ret:
  145. return ret
  146. return 0
  147. if no_toolchains:
  148. toolchains.GetSettings()
  149. toolchains.Scan(options.list_tool_chains and options.verbose)
  150. if options.list_tool_chains:
  151. toolchains.List()
  152. print()
  153. return 0
  154. # Work out how many commits to build. We want to build everything on the
  155. # branch. We also build the upstream commit as a control so we can see
  156. # problems introduced by the first commit on the branch.
  157. count = options.count
  158. has_range = options.branch and '..' in options.branch
  159. if count == -1:
  160. if not options.branch:
  161. count = 1
  162. else:
  163. if has_range:
  164. count, msg = gitutil.CountCommitsInRange(options.git_dir,
  165. options.branch)
  166. else:
  167. count, msg = gitutil.CountCommitsInBranch(options.git_dir,
  168. options.branch)
  169. if count is None:
  170. sys.exit(col.Color(col.RED, msg))
  171. elif count == 0:
  172. sys.exit(col.Color(col.RED, "Range '%s' has no commits" %
  173. options.branch))
  174. if msg:
  175. print(col.Color(col.YELLOW, msg))
  176. count += 1 # Build upstream commit also
  177. if not count:
  178. str = ("No commits found to process in branch '%s': "
  179. "set branch's upstream or use -c flag" % options.branch)
  180. sys.exit(col.Color(col.RED, str))
  181. # Work out what subset of the boards we are building
  182. if not boards:
  183. if not os.path.exists(options.output_dir):
  184. os.makedirs(options.output_dir)
  185. board_file = os.path.join(options.output_dir, 'boards.cfg')
  186. genboardscfg = os.path.join(options.git, 'tools/genboardscfg.py')
  187. status = subprocess.call([genboardscfg, '-o', board_file])
  188. if status != 0:
  189. sys.exit("Failed to generate boards.cfg")
  190. boards = board.Boards()
  191. boards.ReadBoards(board_file)
  192. exclude = []
  193. if options.exclude:
  194. for arg in options.exclude:
  195. exclude += arg.split(',')
  196. if options.boards:
  197. requested_boards = []
  198. for b in options.boards:
  199. requested_boards += b.split(',')
  200. else:
  201. requested_boards = None
  202. why_selected, board_warnings = boards.SelectBoards(args, exclude,
  203. requested_boards)
  204. selected = boards.GetSelected()
  205. if not len(selected):
  206. sys.exit(col.Color(col.RED, 'No matching boards found'))
  207. # Read the metadata from the commits. First look at the upstream commit,
  208. # then the ones in the branch. We would like to do something like
  209. # upstream/master~..branch but that isn't possible if upstream/master is
  210. # a merge commit (it will list all the commits that form part of the
  211. # merge)
  212. # Conflicting tags are not a problem for buildman, since it does not use
  213. # them. For example, Series-version is not useful for buildman. On the
  214. # other hand conflicting tags will cause an error. So allow later tags
  215. # to overwrite earlier ones by setting allow_overwrite=True
  216. if options.branch:
  217. if count == -1:
  218. if has_range:
  219. range_expr = options.branch
  220. else:
  221. range_expr = gitutil.GetRangeInBranch(options.git_dir,
  222. options.branch)
  223. upstream_commit = gitutil.GetUpstream(options.git_dir,
  224. options.branch)
  225. series = patchstream.GetMetaDataForList(upstream_commit,
  226. options.git_dir, 1, series=None, allow_overwrite=True)
  227. series = patchstream.GetMetaDataForList(range_expr,
  228. options.git_dir, None, series, allow_overwrite=True)
  229. else:
  230. # Honour the count
  231. series = patchstream.GetMetaDataForList(options.branch,
  232. options.git_dir, count, series=None, allow_overwrite=True)
  233. else:
  234. series = None
  235. if not options.dry_run:
  236. options.verbose = True
  237. if not options.summary:
  238. options.show_errors = True
  239. # By default we have one thread per CPU. But if there are not enough jobs
  240. # we can have fewer threads and use a high '-j' value for make.
  241. if not options.threads:
  242. options.threads = min(multiprocessing.cpu_count(), len(selected))
  243. if not options.jobs:
  244. options.jobs = max(1, (multiprocessing.cpu_count() +
  245. len(selected) - 1) // len(selected))
  246. if not options.step:
  247. options.step = len(series.commits) - 1
  248. gnu_make = command.Output(os.path.join(options.git,
  249. 'scripts/show-gnu-make'), raise_on_error=False).rstrip()
  250. if not gnu_make:
  251. sys.exit('GNU Make not found')
  252. # Create a new builder with the selected options.
  253. output_dir = options.output_dir
  254. if options.branch:
  255. dirname = options.branch.replace('/', '_')
  256. # As a special case allow the board directory to be placed in the
  257. # output directory itself rather than any subdirectory.
  258. if not options.no_subdirs:
  259. output_dir = os.path.join(options.output_dir, dirname)
  260. if clean_dir and os.path.exists(output_dir):
  261. shutil.rmtree(output_dir)
  262. CheckOutputDir(output_dir)
  263. builder = Builder(toolchains, output_dir, options.git_dir,
  264. options.threads, options.jobs, gnu_make=gnu_make, checkout=True,
  265. show_unknown=options.show_unknown, step=options.step,
  266. no_subdirs=options.no_subdirs, full_path=options.full_path,
  267. verbose_build=options.verbose_build,
  268. incremental=options.incremental,
  269. per_board_out_dir=options.per_board_out_dir,
  270. config_only=options.config_only,
  271. squash_config_y=not options.preserve_config_y,
  272. warnings_as_errors=options.warnings_as_errors)
  273. builder.force_config_on_failure = not options.quick
  274. if make_func:
  275. builder.do_make = make_func
  276. # For a dry run, just show our actions as a sanity check
  277. if options.dry_run:
  278. ShowActions(series, why_selected, selected, builder, options,
  279. board_warnings)
  280. else:
  281. builder.force_build = options.force_build
  282. builder.force_build_failures = options.force_build_failures
  283. builder.force_reconfig = options.force_reconfig
  284. builder.in_tree = options.in_tree
  285. # Work out which boards to build
  286. board_selected = boards.GetSelectedDict()
  287. if series:
  288. commits = series.commits
  289. # Number the commits for test purposes
  290. for commit in range(len(commits)):
  291. commits[commit].sequence = commit
  292. else:
  293. commits = None
  294. Print(GetActionSummary(options.summary, commits, board_selected,
  295. options))
  296. # We can't show function sizes without board details at present
  297. if options.show_bloat:
  298. options.show_detail = True
  299. builder.SetDisplayOptions(options.show_errors, options.show_sizes,
  300. options.show_detail, options.show_bloat,
  301. options.list_error_boards,
  302. options.show_config,
  303. options.show_environment)
  304. if options.summary:
  305. builder.ShowSummary(commits, board_selected)
  306. else:
  307. fail, warned = builder.BuildBoards(commits, board_selected,
  308. options.keep_outputs, options.verbose)
  309. if fail:
  310. return 128
  311. elif warned:
  312. return 129
  313. return 0