control.py 20 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567
  1. # SPDX-License-Identifier: GPL-2.0+
  2. # Copyright (c) 2016 Google, Inc
  3. # Written by Simon Glass <sjg@chromium.org>
  4. #
  5. # Creates binary images from input files controlled by a description
  6. #
  7. from collections import OrderedDict
  8. import glob
  9. import os
  10. import pkg_resources
  11. import sys
  12. from patman import tools
  13. from binman import cbfs_util
  14. from binman import elf
  15. from patman import command
  16. from patman import tout
  17. # List of images we plan to create
  18. # Make this global so that it can be referenced from tests
  19. images = OrderedDict()
  20. def _ReadImageDesc(binman_node):
  21. """Read the image descriptions from the /binman node
  22. This normally produces a single Image object called 'image'. But if
  23. multiple images are present, they will all be returned.
  24. Args:
  25. binman_node: Node object of the /binman node
  26. Returns:
  27. OrderedDict of Image objects, each of which describes an image
  28. """
  29. images = OrderedDict()
  30. if 'multiple-images' in binman_node.props:
  31. for node in binman_node.subnodes:
  32. images[node.name] = Image(node.name, node)
  33. else:
  34. images['image'] = Image('image', binman_node)
  35. return images
  36. def _FindBinmanNode(dtb):
  37. """Find the 'binman' node in the device tree
  38. Args:
  39. dtb: Fdt object to scan
  40. Returns:
  41. Node object of /binman node, or None if not found
  42. """
  43. for node in dtb.GetRoot().subnodes:
  44. if node.name == 'binman':
  45. return node
  46. return None
  47. def GetEntryModules(include_testing=True):
  48. """Get a set of entry class implementations
  49. Returns:
  50. Set of paths to entry class filenames
  51. """
  52. glob_list = pkg_resources.resource_listdir(__name__, 'etype')
  53. glob_list = [fname for fname in glob_list if fname.endswith('.py')]
  54. return set([os.path.splitext(os.path.basename(item))[0]
  55. for item in glob_list
  56. if include_testing or '_testing' not in item])
  57. def WriteEntryDocs(modules, test_missing=None):
  58. """Write out documentation for all entries
  59. Args:
  60. modules: List of Module objects to get docs for
  61. test_missing: Used for testing only, to force an entry's documeentation
  62. to show as missing even if it is present. Should be set to None in
  63. normal use.
  64. """
  65. from binman.entry import Entry
  66. Entry.WriteDocs(modules, test_missing)
  67. def ListEntries(image_fname, entry_paths):
  68. """List the entries in an image
  69. This decodes the supplied image and displays a table of entries from that
  70. image, preceded by a header.
  71. Args:
  72. image_fname: Image filename to process
  73. entry_paths: List of wildcarded paths (e.g. ['*dtb*', 'u-boot*',
  74. 'section/u-boot'])
  75. """
  76. image = Image.FromFile(image_fname)
  77. entries, lines, widths = image.GetListEntries(entry_paths)
  78. num_columns = len(widths)
  79. for linenum, line in enumerate(lines):
  80. if linenum == 1:
  81. # Print header line
  82. print('-' * (sum(widths) + num_columns * 2))
  83. out = ''
  84. for i, item in enumerate(line):
  85. width = -widths[i]
  86. if item.startswith('>'):
  87. width = -width
  88. item = item[1:]
  89. txt = '%*s ' % (width, item)
  90. out += txt
  91. print(out.rstrip())
  92. def ReadEntry(image_fname, entry_path, decomp=True):
  93. """Extract an entry from an image
  94. This extracts the data from a particular entry in an image
  95. Args:
  96. image_fname: Image filename to process
  97. entry_path: Path to entry to extract
  98. decomp: True to return uncompressed data, if the data is compress
  99. False to return the raw data
  100. Returns:
  101. data extracted from the entry
  102. """
  103. global Image
  104. from binman.image import Image
  105. image = Image.FromFile(image_fname)
  106. entry = image.FindEntryPath(entry_path)
  107. return entry.ReadData(decomp)
  108. def ExtractEntries(image_fname, output_fname, outdir, entry_paths,
  109. decomp=True):
  110. """Extract the data from one or more entries and write it to files
  111. Args:
  112. image_fname: Image filename to process
  113. output_fname: Single output filename to use if extracting one file, None
  114. otherwise
  115. outdir: Output directory to use (for any number of files), else None
  116. entry_paths: List of entry paths to extract
  117. decomp: True to decompress the entry data
  118. Returns:
  119. List of EntryInfo records that were written
  120. """
  121. image = Image.FromFile(image_fname)
  122. # Output an entry to a single file, as a special case
  123. if output_fname:
  124. if not entry_paths:
  125. raise ValueError('Must specify an entry path to write with -f')
  126. if len(entry_paths) != 1:
  127. raise ValueError('Must specify exactly one entry path to write with -f')
  128. entry = image.FindEntryPath(entry_paths[0])
  129. data = entry.ReadData(decomp)
  130. tools.WriteFile(output_fname, data)
  131. tout.Notice("Wrote %#x bytes to file '%s'" % (len(data), output_fname))
  132. return
  133. # Otherwise we will output to a path given by the entry path of each entry.
  134. # This means that entries will appear in subdirectories if they are part of
  135. # a sub-section.
  136. einfos = image.GetListEntries(entry_paths)[0]
  137. tout.Notice('%d entries match and will be written' % len(einfos))
  138. for einfo in einfos:
  139. entry = einfo.entry
  140. data = entry.ReadData(decomp)
  141. path = entry.GetPath()[1:]
  142. fname = os.path.join(outdir, path)
  143. # If this entry has children, create a directory for it and put its
  144. # data in a file called 'root' in that directory
  145. if entry.GetEntries():
  146. if not os.path.exists(fname):
  147. os.makedirs(fname)
  148. fname = os.path.join(fname, 'root')
  149. tout.Notice("Write entry '%s' to '%s'" % (entry.GetPath(), fname))
  150. tools.WriteFile(fname, data)
  151. return einfos
  152. def BeforeReplace(image, allow_resize):
  153. """Handle getting an image ready for replacing entries in it
  154. Args:
  155. image: Image to prepare
  156. """
  157. state.PrepareFromLoadedData(image)
  158. image.LoadData()
  159. # If repacking, drop the old offset/size values except for the original
  160. # ones, so we are only left with the constraints.
  161. if allow_resize:
  162. image.ResetForPack()
  163. def ReplaceOneEntry(image, entry, data, do_compress, allow_resize):
  164. """Handle replacing a single entry an an image
  165. Args:
  166. image: Image to update
  167. entry: Entry to write
  168. data: Data to replace with
  169. do_compress: True to compress the data if needed, False if data is
  170. already compressed so should be used as is
  171. allow_resize: True to allow entries to change size (this does a re-pack
  172. of the entries), False to raise an exception
  173. """
  174. if not entry.WriteData(data, do_compress):
  175. if not image.allow_repack:
  176. entry.Raise('Entry data size does not match, but allow-repack is not present for this image')
  177. if not allow_resize:
  178. entry.Raise('Entry data size does not match, but resize is disabled')
  179. def AfterReplace(image, allow_resize, write_map):
  180. """Handle write out an image after replacing entries in it
  181. Args:
  182. image: Image to write
  183. allow_resize: True to allow entries to change size (this does a re-pack
  184. of the entries), False to raise an exception
  185. write_map: True to write a map file
  186. """
  187. tout.Info('Processing image')
  188. ProcessImage(image, update_fdt=True, write_map=write_map,
  189. get_contents=False, allow_resize=allow_resize)
  190. def WriteEntryToImage(image, entry, data, do_compress=True, allow_resize=True,
  191. write_map=False):
  192. BeforeReplace(image, allow_resize)
  193. tout.Info('Writing data to %s' % entry.GetPath())
  194. ReplaceOneEntry(image, entry, data, do_compress, allow_resize)
  195. AfterReplace(image, allow_resize=allow_resize, write_map=write_map)
  196. def WriteEntry(image_fname, entry_path, data, do_compress=True,
  197. allow_resize=True, write_map=False):
  198. """Replace an entry in an image
  199. This replaces the data in a particular entry in an image. This size of the
  200. new data must match the size of the old data unless allow_resize is True.
  201. Args:
  202. image_fname: Image filename to process
  203. entry_path: Path to entry to extract
  204. data: Data to replace with
  205. do_compress: True to compress the data if needed, False if data is
  206. already compressed so should be used as is
  207. allow_resize: True to allow entries to change size (this does a re-pack
  208. of the entries), False to raise an exception
  209. write_map: True to write a map file
  210. Returns:
  211. Image object that was updated
  212. """
  213. tout.Info("Write entry '%s', file '%s'" % (entry_path, image_fname))
  214. image = Image.FromFile(image_fname)
  215. entry = image.FindEntryPath(entry_path)
  216. WriteEntryToImage(image, entry, data, do_compress=do_compress,
  217. allow_resize=allow_resize, write_map=write_map)
  218. return image
  219. def ReplaceEntries(image_fname, input_fname, indir, entry_paths,
  220. do_compress=True, allow_resize=True, write_map=False):
  221. """Replace the data from one or more entries from input files
  222. Args:
  223. image_fname: Image filename to process
  224. input_fname: Single input ilename to use if replacing one file, None
  225. otherwise
  226. indir: Input directory to use (for any number of files), else None
  227. entry_paths: List of entry paths to extract
  228. do_compress: True if the input data is uncompressed and may need to be
  229. compressed if the entry requires it, False if the data is already
  230. compressed.
  231. write_map: True to write a map file
  232. Returns:
  233. List of EntryInfo records that were written
  234. """
  235. image = Image.FromFile(image_fname)
  236. # Replace an entry from a single file, as a special case
  237. if input_fname:
  238. if not entry_paths:
  239. raise ValueError('Must specify an entry path to read with -f')
  240. if len(entry_paths) != 1:
  241. raise ValueError('Must specify exactly one entry path to write with -f')
  242. entry = image.FindEntryPath(entry_paths[0])
  243. data = tools.ReadFile(input_fname)
  244. tout.Notice("Read %#x bytes from file '%s'" % (len(data), input_fname))
  245. WriteEntryToImage(image, entry, data, do_compress=do_compress,
  246. allow_resize=allow_resize, write_map=write_map)
  247. return
  248. # Otherwise we will input from a path given by the entry path of each entry.
  249. # This means that files must appear in subdirectories if they are part of
  250. # a sub-section.
  251. einfos = image.GetListEntries(entry_paths)[0]
  252. tout.Notice("Replacing %d matching entries in image '%s'" %
  253. (len(einfos), image_fname))
  254. BeforeReplace(image, allow_resize)
  255. for einfo in einfos:
  256. entry = einfo.entry
  257. if entry.GetEntries():
  258. tout.Info("Skipping section entry '%s'" % entry.GetPath())
  259. continue
  260. path = entry.GetPath()[1:]
  261. fname = os.path.join(indir, path)
  262. if os.path.exists(fname):
  263. tout.Notice("Write entry '%s' from file '%s'" %
  264. (entry.GetPath(), fname))
  265. data = tools.ReadFile(fname)
  266. ReplaceOneEntry(image, entry, data, do_compress, allow_resize)
  267. else:
  268. tout.Warning("Skipping entry '%s' from missing file '%s'" %
  269. (entry.GetPath(), fname))
  270. AfterReplace(image, allow_resize=allow_resize, write_map=write_map)
  271. return image
  272. def PrepareImagesAndDtbs(dtb_fname, select_images, update_fdt):
  273. """Prepare the images to be processed and select the device tree
  274. This function:
  275. - reads in the device tree
  276. - finds and scans the binman node to create all entries
  277. - selects which images to build
  278. - Updates the device tress with placeholder properties for offset,
  279. image-pos, etc.
  280. Args:
  281. dtb_fname: Filename of the device tree file to use (.dts or .dtb)
  282. selected_images: List of images to output, or None for all
  283. update_fdt: True to update the FDT wth entry offsets, etc.
  284. """
  285. # Import these here in case libfdt.py is not available, in which case
  286. # the above help option still works.
  287. from dtoc import fdt
  288. from dtoc import fdt_util
  289. global images
  290. # Get the device tree ready by compiling it and copying the compiled
  291. # output into a file in our output directly. Then scan it for use
  292. # in binman.
  293. dtb_fname = fdt_util.EnsureCompiled(dtb_fname)
  294. fname = tools.GetOutputFilename('u-boot.dtb.out')
  295. tools.WriteFile(fname, tools.ReadFile(dtb_fname))
  296. dtb = fdt.FdtScan(fname)
  297. node = _FindBinmanNode(dtb)
  298. if not node:
  299. raise ValueError("Device tree '%s' does not have a 'binman' "
  300. "node" % dtb_fname)
  301. images = _ReadImageDesc(node)
  302. if select_images:
  303. skip = []
  304. new_images = OrderedDict()
  305. for name, image in images.items():
  306. if name in select_images:
  307. new_images[name] = image
  308. else:
  309. skip.append(name)
  310. images = new_images
  311. tout.Notice('Skipping images: %s' % ', '.join(skip))
  312. state.Prepare(images, dtb)
  313. # Prepare the device tree by making sure that any missing
  314. # properties are added (e.g. 'pos' and 'size'). The values of these
  315. # may not be correct yet, but we add placeholders so that the
  316. # size of the device tree is correct. Later, in
  317. # SetCalculatedProperties() we will insert the correct values
  318. # without changing the device-tree size, thus ensuring that our
  319. # entry offsets remain the same.
  320. for image in images.values():
  321. image.ExpandEntries()
  322. if update_fdt:
  323. image.AddMissingProperties()
  324. image.ProcessFdt(dtb)
  325. for dtb_item in state.GetAllFdts():
  326. dtb_item.Sync(auto_resize=True)
  327. dtb_item.Pack()
  328. dtb_item.Flush()
  329. return images
  330. def ProcessImage(image, update_fdt, write_map, get_contents=True,
  331. allow_resize=True, allow_missing=False):
  332. """Perform all steps for this image, including checking and # writing it.
  333. This means that errors found with a later image will be reported after
  334. earlier images are already completed and written, but that does not seem
  335. important.
  336. Args:
  337. image: Image to process
  338. update_fdt: True to update the FDT wth entry offsets, etc.
  339. write_map: True to write a map file
  340. get_contents: True to get the image contents from files, etc., False if
  341. the contents is already present
  342. allow_resize: True to allow entries to change size (this does a re-pack
  343. of the entries), False to raise an exception
  344. allow_missing: Allow blob_ext objects to be missing
  345. Returns:
  346. True if one or more external blobs are missing, False if all are present
  347. """
  348. if get_contents:
  349. image.SetAllowMissing(allow_missing)
  350. image.GetEntryContents()
  351. image.GetEntryOffsets()
  352. # We need to pack the entries to figure out where everything
  353. # should be placed. This sets the offset/size of each entry.
  354. # However, after packing we call ProcessEntryContents() which
  355. # may result in an entry changing size. In that case we need to
  356. # do another pass. Since the device tree often contains the
  357. # final offset/size information we try to make space for this in
  358. # AddMissingProperties() above. However, if the device is
  359. # compressed we cannot know this compressed size in advance,
  360. # since changing an offset from 0x100 to 0x104 (for example) can
  361. # alter the compressed size of the device tree. So we need a
  362. # third pass for this.
  363. passes = 5
  364. for pack_pass in range(passes):
  365. try:
  366. image.PackEntries()
  367. image.CheckSize()
  368. image.CheckEntries()
  369. except Exception as e:
  370. if write_map:
  371. fname = image.WriteMap()
  372. print("Wrote map file '%s' to show errors" % fname)
  373. raise
  374. image.SetImagePos()
  375. if update_fdt:
  376. image.SetCalculatedProperties()
  377. for dtb_item in state.GetAllFdts():
  378. dtb_item.Sync()
  379. dtb_item.Flush()
  380. image.WriteSymbols()
  381. sizes_ok = image.ProcessEntryContents()
  382. if sizes_ok:
  383. break
  384. image.ResetForPack()
  385. tout.Info('Pack completed after %d pass(es)' % (pack_pass + 1))
  386. if not sizes_ok:
  387. image.Raise('Entries changed size after packing (tried %s passes)' %
  388. passes)
  389. image.BuildImage()
  390. if write_map:
  391. image.WriteMap()
  392. missing_list = []
  393. image.CheckMissing(missing_list)
  394. if missing_list:
  395. tout.Warning("Image '%s' is missing external blobs and is non-functional: %s" %
  396. (image.name, ' '.join([e.name for e in missing_list])))
  397. return bool(missing_list)
  398. def Binman(args):
  399. """The main control code for binman
  400. This assumes that help and test options have already been dealt with. It
  401. deals with the core task of building images.
  402. Args:
  403. args: Command line arguments Namespace object
  404. """
  405. global Image
  406. global state
  407. if args.full_help:
  408. pager = os.getenv('PAGER')
  409. if not pager:
  410. pager = 'more'
  411. fname = os.path.join(os.path.dirname(os.path.realpath(sys.argv[0])),
  412. 'README')
  413. command.Run(pager, fname)
  414. return 0
  415. # Put these here so that we can import this module without libfdt
  416. from binman.image import Image
  417. from binman import state
  418. if args.cmd in ['ls', 'extract', 'replace']:
  419. try:
  420. tout.Init(args.verbosity)
  421. tools.PrepareOutputDir(None)
  422. if args.cmd == 'ls':
  423. ListEntries(args.image, args.paths)
  424. if args.cmd == 'extract':
  425. ExtractEntries(args.image, args.filename, args.outdir, args.paths,
  426. not args.uncompressed)
  427. if args.cmd == 'replace':
  428. ReplaceEntries(args.image, args.filename, args.indir, args.paths,
  429. do_compress=not args.compressed,
  430. allow_resize=not args.fix_size, write_map=args.map)
  431. except:
  432. raise
  433. finally:
  434. tools.FinaliseOutputDir()
  435. return 0
  436. # Try to figure out which device tree contains our image description
  437. if args.dt:
  438. dtb_fname = args.dt
  439. else:
  440. board = args.board
  441. if not board:
  442. raise ValueError('Must provide a board to process (use -b <board>)')
  443. board_pathname = os.path.join(args.build_dir, board)
  444. dtb_fname = os.path.join(board_pathname, 'u-boot.dtb')
  445. if not args.indir:
  446. args.indir = ['.']
  447. args.indir.append(board_pathname)
  448. try:
  449. tout.Init(args.verbosity)
  450. elf.debug = args.debug
  451. cbfs_util.VERBOSE = args.verbosity > 2
  452. state.use_fake_dtb = args.fake_dtb
  453. try:
  454. tools.SetInputDirs(args.indir)
  455. tools.PrepareOutputDir(args.outdir, args.preserve)
  456. tools.SetToolPaths(args.toolpath)
  457. state.SetEntryArgs(args.entry_arg)
  458. images = PrepareImagesAndDtbs(dtb_fname, args.image,
  459. args.update_fdt)
  460. missing = False
  461. for image in images.values():
  462. missing |= ProcessImage(image, args.update_fdt, args.map,
  463. allow_missing=args.allow_missing)
  464. # Write the updated FDTs to our output files
  465. for dtb_item in state.GetAllFdts():
  466. tools.WriteFile(dtb_item._fname, dtb_item.GetContents())
  467. if missing:
  468. tout.Warning("Some images are invalid")
  469. finally:
  470. tools.FinaliseOutputDir()
  471. finally:
  472. tout.Uninit()
  473. return 0