control.py 23 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657
  1. # SPDX-License-Identifier: GPL-2.0+
  2. # Copyright (c) 2016 Google, Inc
  3. # Written by Simon Glass <sjg@chromium.org>
  4. #
  5. # Creates binary images from input files controlled by a description
  6. #
  7. from collections import OrderedDict
  8. import glob
  9. import os
  10. import pkg_resources
  11. import re
  12. import sys
  13. from patman import tools
  14. from binman import cbfs_util
  15. from binman import elf
  16. from patman import command
  17. from patman import tout
  18. # List of images we plan to create
  19. # Make this global so that it can be referenced from tests
  20. images = OrderedDict()
  21. # Help text for each type of missing blob, dict:
  22. # key: Value of the entry's 'missing-msg' or entry name
  23. # value: Text for the help
  24. missing_blob_help = {}
  25. def _ReadImageDesc(binman_node, use_expanded):
  26. """Read the image descriptions from the /binman node
  27. This normally produces a single Image object called 'image'. But if
  28. multiple images are present, they will all be returned.
  29. Args:
  30. binman_node: Node object of the /binman node
  31. use_expanded: True if the FDT will be updated with the entry information
  32. Returns:
  33. OrderedDict of Image objects, each of which describes an image
  34. """
  35. images = OrderedDict()
  36. if 'multiple-images' in binman_node.props:
  37. for node in binman_node.subnodes:
  38. images[node.name] = Image(node.name, node,
  39. use_expanded=use_expanded)
  40. else:
  41. images['image'] = Image('image', binman_node, use_expanded=use_expanded)
  42. return images
  43. def _FindBinmanNode(dtb):
  44. """Find the 'binman' node in the device tree
  45. Args:
  46. dtb: Fdt object to scan
  47. Returns:
  48. Node object of /binman node, or None if not found
  49. """
  50. for node in dtb.GetRoot().subnodes:
  51. if node.name == 'binman':
  52. return node
  53. return None
  54. def _ReadMissingBlobHelp():
  55. """Read the missing-blob-help file
  56. This file containins help messages explaining what to do when external blobs
  57. are missing.
  58. Returns:
  59. Dict:
  60. key: Message tag (str)
  61. value: Message text (str)
  62. """
  63. def _FinishTag(tag, msg, result):
  64. if tag:
  65. result[tag] = msg.rstrip()
  66. tag = None
  67. msg = ''
  68. return tag, msg
  69. my_data = pkg_resources.resource_string(__name__, 'missing-blob-help')
  70. re_tag = re.compile('^([-a-z0-9]+):$')
  71. result = {}
  72. tag = None
  73. msg = ''
  74. for line in my_data.decode('utf-8').splitlines():
  75. if not line.startswith('#'):
  76. m_tag = re_tag.match(line)
  77. if m_tag:
  78. _, msg = _FinishTag(tag, msg, result)
  79. tag = m_tag.group(1)
  80. elif tag:
  81. msg += line + '\n'
  82. _FinishTag(tag, msg, result)
  83. return result
  84. def _ShowBlobHelp(path, text):
  85. tout.Warning('\n%s:' % path)
  86. for line in text.splitlines():
  87. tout.Warning(' %s' % line)
  88. def _ShowHelpForMissingBlobs(missing_list):
  89. """Show help for each missing blob to help the user take action
  90. Args:
  91. missing_list: List of Entry objects to show help for
  92. """
  93. global missing_blob_help
  94. if not missing_blob_help:
  95. missing_blob_help = _ReadMissingBlobHelp()
  96. for entry in missing_list:
  97. tags = entry.GetHelpTags()
  98. # Show the first match help message
  99. for tag in tags:
  100. if tag in missing_blob_help:
  101. _ShowBlobHelp(entry._node.path, missing_blob_help[tag])
  102. break
  103. def GetEntryModules(include_testing=True):
  104. """Get a set of entry class implementations
  105. Returns:
  106. Set of paths to entry class filenames
  107. """
  108. glob_list = pkg_resources.resource_listdir(__name__, 'etype')
  109. glob_list = [fname for fname in glob_list if fname.endswith('.py')]
  110. return set([os.path.splitext(os.path.basename(item))[0]
  111. for item in glob_list
  112. if include_testing or '_testing' not in item])
  113. def WriteEntryDocs(modules, test_missing=None):
  114. """Write out documentation for all entries
  115. Args:
  116. modules: List of Module objects to get docs for
  117. test_missing: Used for testing only, to force an entry's documeentation
  118. to show as missing even if it is present. Should be set to None in
  119. normal use.
  120. """
  121. from binman.entry import Entry
  122. Entry.WriteDocs(modules, test_missing)
  123. def ListEntries(image_fname, entry_paths):
  124. """List the entries in an image
  125. This decodes the supplied image and displays a table of entries from that
  126. image, preceded by a header.
  127. Args:
  128. image_fname: Image filename to process
  129. entry_paths: List of wildcarded paths (e.g. ['*dtb*', 'u-boot*',
  130. 'section/u-boot'])
  131. """
  132. image = Image.FromFile(image_fname)
  133. entries, lines, widths = image.GetListEntries(entry_paths)
  134. num_columns = len(widths)
  135. for linenum, line in enumerate(lines):
  136. if linenum == 1:
  137. # Print header line
  138. print('-' * (sum(widths) + num_columns * 2))
  139. out = ''
  140. for i, item in enumerate(line):
  141. width = -widths[i]
  142. if item.startswith('>'):
  143. width = -width
  144. item = item[1:]
  145. txt = '%*s ' % (width, item)
  146. out += txt
  147. print(out.rstrip())
  148. def ReadEntry(image_fname, entry_path, decomp=True):
  149. """Extract an entry from an image
  150. This extracts the data from a particular entry in an image
  151. Args:
  152. image_fname: Image filename to process
  153. entry_path: Path to entry to extract
  154. decomp: True to return uncompressed data, if the data is compress
  155. False to return the raw data
  156. Returns:
  157. data extracted from the entry
  158. """
  159. global Image
  160. from binman.image import Image
  161. image = Image.FromFile(image_fname)
  162. entry = image.FindEntryPath(entry_path)
  163. return entry.ReadData(decomp)
  164. def ExtractEntries(image_fname, output_fname, outdir, entry_paths,
  165. decomp=True):
  166. """Extract the data from one or more entries and write it to files
  167. Args:
  168. image_fname: Image filename to process
  169. output_fname: Single output filename to use if extracting one file, None
  170. otherwise
  171. outdir: Output directory to use (for any number of files), else None
  172. entry_paths: List of entry paths to extract
  173. decomp: True to decompress the entry data
  174. Returns:
  175. List of EntryInfo records that were written
  176. """
  177. image = Image.FromFile(image_fname)
  178. # Output an entry to a single file, as a special case
  179. if output_fname:
  180. if not entry_paths:
  181. raise ValueError('Must specify an entry path to write with -f')
  182. if len(entry_paths) != 1:
  183. raise ValueError('Must specify exactly one entry path to write with -f')
  184. entry = image.FindEntryPath(entry_paths[0])
  185. data = entry.ReadData(decomp)
  186. tools.WriteFile(output_fname, data)
  187. tout.Notice("Wrote %#x bytes to file '%s'" % (len(data), output_fname))
  188. return
  189. # Otherwise we will output to a path given by the entry path of each entry.
  190. # This means that entries will appear in subdirectories if they are part of
  191. # a sub-section.
  192. einfos = image.GetListEntries(entry_paths)[0]
  193. tout.Notice('%d entries match and will be written' % len(einfos))
  194. for einfo in einfos:
  195. entry = einfo.entry
  196. data = entry.ReadData(decomp)
  197. path = entry.GetPath()[1:]
  198. fname = os.path.join(outdir, path)
  199. # If this entry has children, create a directory for it and put its
  200. # data in a file called 'root' in that directory
  201. if entry.GetEntries():
  202. if fname and not os.path.exists(fname):
  203. os.makedirs(fname)
  204. fname = os.path.join(fname, 'root')
  205. tout.Notice("Write entry '%s' size %x to '%s'" %
  206. (entry.GetPath(), len(data), fname))
  207. tools.WriteFile(fname, data)
  208. return einfos
  209. def BeforeReplace(image, allow_resize):
  210. """Handle getting an image ready for replacing entries in it
  211. Args:
  212. image: Image to prepare
  213. """
  214. state.PrepareFromLoadedData(image)
  215. image.LoadData()
  216. # If repacking, drop the old offset/size values except for the original
  217. # ones, so we are only left with the constraints.
  218. if allow_resize:
  219. image.ResetForPack()
  220. def ReplaceOneEntry(image, entry, data, do_compress, allow_resize):
  221. """Handle replacing a single entry an an image
  222. Args:
  223. image: Image to update
  224. entry: Entry to write
  225. data: Data to replace with
  226. do_compress: True to compress the data if needed, False if data is
  227. already compressed so should be used as is
  228. allow_resize: True to allow entries to change size (this does a re-pack
  229. of the entries), False to raise an exception
  230. """
  231. if not entry.WriteData(data, do_compress):
  232. if not image.allow_repack:
  233. entry.Raise('Entry data size does not match, but allow-repack is not present for this image')
  234. if not allow_resize:
  235. entry.Raise('Entry data size does not match, but resize is disabled')
  236. def AfterReplace(image, allow_resize, write_map):
  237. """Handle write out an image after replacing entries in it
  238. Args:
  239. image: Image to write
  240. allow_resize: True to allow entries to change size (this does a re-pack
  241. of the entries), False to raise an exception
  242. write_map: True to write a map file
  243. """
  244. tout.Info('Processing image')
  245. ProcessImage(image, update_fdt=True, write_map=write_map,
  246. get_contents=False, allow_resize=allow_resize)
  247. def WriteEntryToImage(image, entry, data, do_compress=True, allow_resize=True,
  248. write_map=False):
  249. BeforeReplace(image, allow_resize)
  250. tout.Info('Writing data to %s' % entry.GetPath())
  251. ReplaceOneEntry(image, entry, data, do_compress, allow_resize)
  252. AfterReplace(image, allow_resize=allow_resize, write_map=write_map)
  253. def WriteEntry(image_fname, entry_path, data, do_compress=True,
  254. allow_resize=True, write_map=False):
  255. """Replace an entry in an image
  256. This replaces the data in a particular entry in an image. This size of the
  257. new data must match the size of the old data unless allow_resize is True.
  258. Args:
  259. image_fname: Image filename to process
  260. entry_path: Path to entry to extract
  261. data: Data to replace with
  262. do_compress: True to compress the data if needed, False if data is
  263. already compressed so should be used as is
  264. allow_resize: True to allow entries to change size (this does a re-pack
  265. of the entries), False to raise an exception
  266. write_map: True to write a map file
  267. Returns:
  268. Image object that was updated
  269. """
  270. tout.Info("Write entry '%s', file '%s'" % (entry_path, image_fname))
  271. image = Image.FromFile(image_fname)
  272. entry = image.FindEntryPath(entry_path)
  273. WriteEntryToImage(image, entry, data, do_compress=do_compress,
  274. allow_resize=allow_resize, write_map=write_map)
  275. return image
  276. def ReplaceEntries(image_fname, input_fname, indir, entry_paths,
  277. do_compress=True, allow_resize=True, write_map=False):
  278. """Replace the data from one or more entries from input files
  279. Args:
  280. image_fname: Image filename to process
  281. input_fname: Single input ilename to use if replacing one file, None
  282. otherwise
  283. indir: Input directory to use (for any number of files), else None
  284. entry_paths: List of entry paths to extract
  285. do_compress: True if the input data is uncompressed and may need to be
  286. compressed if the entry requires it, False if the data is already
  287. compressed.
  288. write_map: True to write a map file
  289. Returns:
  290. List of EntryInfo records that were written
  291. """
  292. image = Image.FromFile(image_fname)
  293. # Replace an entry from a single file, as a special case
  294. if input_fname:
  295. if not entry_paths:
  296. raise ValueError('Must specify an entry path to read with -f')
  297. if len(entry_paths) != 1:
  298. raise ValueError('Must specify exactly one entry path to write with -f')
  299. entry = image.FindEntryPath(entry_paths[0])
  300. data = tools.ReadFile(input_fname)
  301. tout.Notice("Read %#x bytes from file '%s'" % (len(data), input_fname))
  302. WriteEntryToImage(image, entry, data, do_compress=do_compress,
  303. allow_resize=allow_resize, write_map=write_map)
  304. return
  305. # Otherwise we will input from a path given by the entry path of each entry.
  306. # This means that files must appear in subdirectories if they are part of
  307. # a sub-section.
  308. einfos = image.GetListEntries(entry_paths)[0]
  309. tout.Notice("Replacing %d matching entries in image '%s'" %
  310. (len(einfos), image_fname))
  311. BeforeReplace(image, allow_resize)
  312. for einfo in einfos:
  313. entry = einfo.entry
  314. if entry.GetEntries():
  315. tout.Info("Skipping section entry '%s'" % entry.GetPath())
  316. continue
  317. path = entry.GetPath()[1:]
  318. fname = os.path.join(indir, path)
  319. if os.path.exists(fname):
  320. tout.Notice("Write entry '%s' from file '%s'" %
  321. (entry.GetPath(), fname))
  322. data = tools.ReadFile(fname)
  323. ReplaceOneEntry(image, entry, data, do_compress, allow_resize)
  324. else:
  325. tout.Warning("Skipping entry '%s' from missing file '%s'" %
  326. (entry.GetPath(), fname))
  327. AfterReplace(image, allow_resize=allow_resize, write_map=write_map)
  328. return image
  329. def PrepareImagesAndDtbs(dtb_fname, select_images, update_fdt, use_expanded):
  330. """Prepare the images to be processed and select the device tree
  331. This function:
  332. - reads in the device tree
  333. - finds and scans the binman node to create all entries
  334. - selects which images to build
  335. - Updates the device tress with placeholder properties for offset,
  336. image-pos, etc.
  337. Args:
  338. dtb_fname: Filename of the device tree file to use (.dts or .dtb)
  339. selected_images: List of images to output, or None for all
  340. update_fdt: True to update the FDT wth entry offsets, etc.
  341. use_expanded: True to use expanded versions of entries, if available.
  342. So if 'u-boot' is called for, we use 'u-boot-expanded' instead. This
  343. is needed if update_fdt is True (although tests may disable it)
  344. Returns:
  345. OrderedDict of images:
  346. key: Image name (str)
  347. value: Image object
  348. """
  349. # Import these here in case libfdt.py is not available, in which case
  350. # the above help option still works.
  351. from dtoc import fdt
  352. from dtoc import fdt_util
  353. global images
  354. # Get the device tree ready by compiling it and copying the compiled
  355. # output into a file in our output directly. Then scan it for use
  356. # in binman.
  357. dtb_fname = fdt_util.EnsureCompiled(dtb_fname)
  358. fname = tools.GetOutputFilename('u-boot.dtb.out')
  359. tools.WriteFile(fname, tools.ReadFile(dtb_fname))
  360. dtb = fdt.FdtScan(fname)
  361. node = _FindBinmanNode(dtb)
  362. if not node:
  363. raise ValueError("Device tree '%s' does not have a 'binman' "
  364. "node" % dtb_fname)
  365. images = _ReadImageDesc(node, use_expanded)
  366. if select_images:
  367. skip = []
  368. new_images = OrderedDict()
  369. for name, image in images.items():
  370. if name in select_images:
  371. new_images[name] = image
  372. else:
  373. skip.append(name)
  374. images = new_images
  375. tout.Notice('Skipping images: %s' % ', '.join(skip))
  376. state.Prepare(images, dtb)
  377. # Prepare the device tree by making sure that any missing
  378. # properties are added (e.g. 'pos' and 'size'). The values of these
  379. # may not be correct yet, but we add placeholders so that the
  380. # size of the device tree is correct. Later, in
  381. # SetCalculatedProperties() we will insert the correct values
  382. # without changing the device-tree size, thus ensuring that our
  383. # entry offsets remain the same.
  384. for image in images.values():
  385. image.ExpandEntries()
  386. if update_fdt:
  387. image.AddMissingProperties(True)
  388. image.ProcessFdt(dtb)
  389. for dtb_item in state.GetAllFdts():
  390. dtb_item.Sync(auto_resize=True)
  391. dtb_item.Pack()
  392. dtb_item.Flush()
  393. return images
  394. def ProcessImage(image, update_fdt, write_map, get_contents=True,
  395. allow_resize=True, allow_missing=False):
  396. """Perform all steps for this image, including checking and # writing it.
  397. This means that errors found with a later image will be reported after
  398. earlier images are already completed and written, but that does not seem
  399. important.
  400. Args:
  401. image: Image to process
  402. update_fdt: True to update the FDT wth entry offsets, etc.
  403. write_map: True to write a map file
  404. get_contents: True to get the image contents from files, etc., False if
  405. the contents is already present
  406. allow_resize: True to allow entries to change size (this does a re-pack
  407. of the entries), False to raise an exception
  408. allow_missing: Allow blob_ext objects to be missing
  409. Returns:
  410. True if one or more external blobs are missing, False if all are present
  411. """
  412. if get_contents:
  413. image.SetAllowMissing(allow_missing)
  414. image.GetEntryContents()
  415. image.GetEntryOffsets()
  416. # We need to pack the entries to figure out where everything
  417. # should be placed. This sets the offset/size of each entry.
  418. # However, after packing we call ProcessEntryContents() which
  419. # may result in an entry changing size. In that case we need to
  420. # do another pass. Since the device tree often contains the
  421. # final offset/size information we try to make space for this in
  422. # AddMissingProperties() above. However, if the device is
  423. # compressed we cannot know this compressed size in advance,
  424. # since changing an offset from 0x100 to 0x104 (for example) can
  425. # alter the compressed size of the device tree. So we need a
  426. # third pass for this.
  427. passes = 5
  428. for pack_pass in range(passes):
  429. try:
  430. image.PackEntries()
  431. except Exception as e:
  432. if write_map:
  433. fname = image.WriteMap()
  434. print("Wrote map file '%s' to show errors" % fname)
  435. raise
  436. image.SetImagePos()
  437. if update_fdt:
  438. image.SetCalculatedProperties()
  439. for dtb_item in state.GetAllFdts():
  440. dtb_item.Sync()
  441. dtb_item.Flush()
  442. image.WriteSymbols()
  443. sizes_ok = image.ProcessEntryContents()
  444. if sizes_ok:
  445. break
  446. image.ResetForPack()
  447. tout.Info('Pack completed after %d pass(es)' % (pack_pass + 1))
  448. if not sizes_ok:
  449. image.Raise('Entries changed size after packing (tried %s passes)' %
  450. passes)
  451. image.BuildImage()
  452. if write_map:
  453. image.WriteMap()
  454. missing_list = []
  455. image.CheckMissing(missing_list)
  456. if missing_list:
  457. tout.Warning("Image '%s' is missing external blobs and is non-functional: %s" %
  458. (image.name, ' '.join([e.name for e in missing_list])))
  459. _ShowHelpForMissingBlobs(missing_list)
  460. return bool(missing_list)
  461. def Binman(args):
  462. """The main control code for binman
  463. This assumes that help and test options have already been dealt with. It
  464. deals with the core task of building images.
  465. Args:
  466. args: Command line arguments Namespace object
  467. """
  468. global Image
  469. global state
  470. if args.full_help:
  471. pager = os.getenv('PAGER')
  472. if not pager:
  473. pager = 'more'
  474. fname = os.path.join(os.path.dirname(os.path.realpath(sys.argv[0])),
  475. 'README.rst')
  476. command.Run(pager, fname)
  477. return 0
  478. # Put these here so that we can import this module without libfdt
  479. from binman.image import Image
  480. from binman import state
  481. if args.cmd in ['ls', 'extract', 'replace']:
  482. try:
  483. tout.Init(args.verbosity)
  484. tools.PrepareOutputDir(None)
  485. if args.cmd == 'ls':
  486. ListEntries(args.image, args.paths)
  487. if args.cmd == 'extract':
  488. ExtractEntries(args.image, args.filename, args.outdir, args.paths,
  489. not args.uncompressed)
  490. if args.cmd == 'replace':
  491. ReplaceEntries(args.image, args.filename, args.indir, args.paths,
  492. do_compress=not args.compressed,
  493. allow_resize=not args.fix_size, write_map=args.map)
  494. except:
  495. raise
  496. finally:
  497. tools.FinaliseOutputDir()
  498. return 0
  499. # Try to figure out which device tree contains our image description
  500. if args.dt:
  501. dtb_fname = args.dt
  502. else:
  503. board = args.board
  504. if not board:
  505. raise ValueError('Must provide a board to process (use -b <board>)')
  506. board_pathname = os.path.join(args.build_dir, board)
  507. dtb_fname = os.path.join(board_pathname, 'u-boot.dtb')
  508. if not args.indir:
  509. args.indir = ['.']
  510. args.indir.append(board_pathname)
  511. try:
  512. tout.Init(args.verbosity)
  513. elf.debug = args.debug
  514. cbfs_util.VERBOSE = args.verbosity > 2
  515. state.use_fake_dtb = args.fake_dtb
  516. # Normally we replace the 'u-boot' etype with 'u-boot-expanded', etc.
  517. # When running tests this can be disabled using this flag. When not
  518. # updating the FDT in image, it is not needed by binman, but we use it
  519. # for consistency, so that the images look the same to U-Boot at
  520. # runtime.
  521. use_expanded = not args.no_expanded
  522. try:
  523. tools.SetInputDirs(args.indir)
  524. tools.PrepareOutputDir(args.outdir, args.preserve)
  525. tools.SetToolPaths(args.toolpath)
  526. state.SetEntryArgs(args.entry_arg)
  527. state.SetThreads(args.threads)
  528. images = PrepareImagesAndDtbs(dtb_fname, args.image,
  529. args.update_fdt, use_expanded)
  530. if args.test_section_timeout:
  531. # Set the first image to timeout, used in testThreadTimeout()
  532. images[list(images.keys())[0]].test_section_timeout = True
  533. missing = False
  534. for image in images.values():
  535. missing |= ProcessImage(image, args.update_fdt, args.map,
  536. allow_missing=args.allow_missing)
  537. # Write the updated FDTs to our output files
  538. for dtb_item in state.GetAllFdts():
  539. tools.WriteFile(dtb_item._fname, dtb_item.GetContents())
  540. if missing:
  541. tout.Warning("\nSome images are invalid")
  542. # Use this to debug the time take to pack the image
  543. #state.TimingShow()
  544. finally:
  545. tools.FinaliseOutputDir()
  546. finally:
  547. tout.Uninit()
  548. return 0