tools.py 18 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583
  1. # SPDX-License-Identifier: GPL-2.0+
  2. #
  3. # Copyright (c) 2016 Google, Inc
  4. #
  5. import glob
  6. import os
  7. import shutil
  8. import struct
  9. import sys
  10. import tempfile
  11. from patman import command
  12. from patman import tout
  13. # Output directly (generally this is temporary)
  14. outdir = None
  15. # True to keep the output directory around after exiting
  16. preserve_outdir = False
  17. # Path to the Chrome OS chroot, if we know it
  18. chroot_path = None
  19. # Search paths to use for Filename(), used to find files
  20. search_paths = []
  21. tool_search_paths = []
  22. # Tools and the packages that contain them, on debian
  23. packages = {
  24. 'lz4': 'liblz4-tool',
  25. }
  26. # List of paths to use when looking for an input file
  27. indir = []
  28. def PrepareOutputDir(dirname, preserve=False):
  29. """Select an output directory, ensuring it exists.
  30. This either creates a temporary directory or checks that the one supplied
  31. by the user is valid. For a temporary directory, it makes a note to
  32. remove it later if required.
  33. Args:
  34. dirname: a string, name of the output directory to use to store
  35. intermediate and output files. If is None - create a temporary
  36. directory.
  37. preserve: a Boolean. If outdir above is None and preserve is False, the
  38. created temporary directory will be destroyed on exit.
  39. Raises:
  40. OSError: If it cannot create the output directory.
  41. """
  42. global outdir, preserve_outdir
  43. preserve_outdir = dirname or preserve
  44. if dirname:
  45. outdir = dirname
  46. if not os.path.isdir(outdir):
  47. try:
  48. os.makedirs(outdir)
  49. except OSError as err:
  50. raise CmdError("Cannot make output directory '%s': '%s'" %
  51. (outdir, err.strerror))
  52. tout.Debug("Using output directory '%s'" % outdir)
  53. else:
  54. outdir = tempfile.mkdtemp(prefix='binman.')
  55. tout.Debug("Using temporary directory '%s'" % outdir)
  56. def _RemoveOutputDir():
  57. global outdir
  58. shutil.rmtree(outdir)
  59. tout.Debug("Deleted temporary directory '%s'" % outdir)
  60. outdir = None
  61. def FinaliseOutputDir():
  62. global outdir, preserve_outdir
  63. """Tidy up: delete output directory if temporary and not preserved."""
  64. if outdir and not preserve_outdir:
  65. _RemoveOutputDir()
  66. outdir = None
  67. def GetOutputFilename(fname):
  68. """Return a filename within the output directory.
  69. Args:
  70. fname: Filename to use for new file
  71. Returns:
  72. The full path of the filename, within the output directory
  73. """
  74. return os.path.join(outdir, fname)
  75. def GetOutputDir():
  76. """Return the current output directory
  77. Returns:
  78. str: The output directory
  79. """
  80. return outdir
  81. def _FinaliseForTest():
  82. """Remove the output directory (for use by tests)"""
  83. global outdir
  84. if outdir:
  85. _RemoveOutputDir()
  86. outdir = None
  87. def SetInputDirs(dirname):
  88. """Add a list of input directories, where input files are kept.
  89. Args:
  90. dirname: a list of paths to input directories to use for obtaining
  91. files needed by binman to place in the image.
  92. """
  93. global indir
  94. indir = dirname
  95. tout.Debug("Using input directories %s" % indir)
  96. def GetInputFilename(fname, allow_missing=False):
  97. """Return a filename for use as input.
  98. Args:
  99. fname: Filename to use for new file
  100. allow_missing: True if the filename can be missing
  101. Returns:
  102. fname, if indir is None;
  103. full path of the filename, within the input directory;
  104. None, if file is missing and allow_missing is True
  105. Raises:
  106. ValueError if file is missing and allow_missing is False
  107. """
  108. if not indir or fname[:1] == '/':
  109. return fname
  110. for dirname in indir:
  111. pathname = os.path.join(dirname, fname)
  112. if os.path.exists(pathname):
  113. return pathname
  114. if allow_missing:
  115. return None
  116. raise ValueError("Filename '%s' not found in input path (%s) (cwd='%s')" %
  117. (fname, ','.join(indir), os.getcwd()))
  118. def GetInputFilenameGlob(pattern):
  119. """Return a list of filenames for use as input.
  120. Args:
  121. pattern: Filename pattern to search for
  122. Returns:
  123. A list of matching files in all input directories
  124. """
  125. if not indir:
  126. return glob.glob(fname)
  127. files = []
  128. for dirname in indir:
  129. pathname = os.path.join(dirname, pattern)
  130. files += glob.glob(pathname)
  131. return sorted(files)
  132. def Align(pos, align):
  133. if align:
  134. mask = align - 1
  135. pos = (pos + mask) & ~mask
  136. return pos
  137. def NotPowerOfTwo(num):
  138. return num and (num & (num - 1))
  139. def SetToolPaths(toolpaths):
  140. """Set the path to search for tools
  141. Args:
  142. toolpaths: List of paths to search for tools executed by Run()
  143. """
  144. global tool_search_paths
  145. tool_search_paths = toolpaths
  146. def PathHasFile(path_spec, fname):
  147. """Check if a given filename is in the PATH
  148. Args:
  149. path_spec: Value of PATH variable to check
  150. fname: Filename to check
  151. Returns:
  152. True if found, False if not
  153. """
  154. for dir in path_spec.split(':'):
  155. if os.path.exists(os.path.join(dir, fname)):
  156. return True
  157. return False
  158. def GetHostCompileTool(name):
  159. """Get the host-specific version for a compile tool
  160. This checks the environment variables that specify which version of
  161. the tool should be used (e.g. ${HOSTCC}).
  162. The following table lists the host-specific versions of the tools
  163. this function resolves to:
  164. Compile Tool | Host version
  165. --------------+----------------
  166. as | ${HOSTAS}
  167. ld | ${HOSTLD}
  168. cc | ${HOSTCC}
  169. cpp | ${HOSTCPP}
  170. c++ | ${HOSTCXX}
  171. ar | ${HOSTAR}
  172. nm | ${HOSTNM}
  173. ldr | ${HOSTLDR}
  174. strip | ${HOSTSTRIP}
  175. objcopy | ${HOSTOBJCOPY}
  176. objdump | ${HOSTOBJDUMP}
  177. dtc | ${HOSTDTC}
  178. Args:
  179. name: Command name to run
  180. Returns:
  181. host_name: Exact command name to run instead
  182. extra_args: List of extra arguments to pass
  183. """
  184. host_name = None
  185. extra_args = []
  186. if name in ('as', 'ld', 'cc', 'cpp', 'ar', 'nm', 'ldr', 'strip',
  187. 'objcopy', 'objdump', 'dtc'):
  188. host_name, *host_args = env.get('HOST' + name.upper(), '').split(' ')
  189. elif name == 'c++':
  190. host_name, *host_args = env.get('HOSTCXX', '').split(' ')
  191. if host_name:
  192. return host_name, extra_args
  193. return name, []
  194. def GetTargetCompileTool(name, cross_compile=None):
  195. """Get the target-specific version for a compile tool
  196. This first checks the environment variables that specify which
  197. version of the tool should be used (e.g. ${CC}). If those aren't
  198. specified, it checks the CROSS_COMPILE variable as a prefix for the
  199. tool with some substitutions (e.g. "${CROSS_COMPILE}gcc" for cc).
  200. The following table lists the target-specific versions of the tools
  201. this function resolves to:
  202. Compile Tool | First choice | Second choice
  203. --------------+----------------+----------------------------
  204. as | ${AS} | ${CROSS_COMPILE}as
  205. ld | ${LD} | ${CROSS_COMPILE}ld.bfd
  206. | | or ${CROSS_COMPILE}ld
  207. cc | ${CC} | ${CROSS_COMPILE}gcc
  208. cpp | ${CPP} | ${CROSS_COMPILE}gcc -E
  209. c++ | ${CXX} | ${CROSS_COMPILE}g++
  210. ar | ${AR} | ${CROSS_COMPILE}ar
  211. nm | ${NM} | ${CROSS_COMPILE}nm
  212. ldr | ${LDR} | ${CROSS_COMPILE}ldr
  213. strip | ${STRIP} | ${CROSS_COMPILE}strip
  214. objcopy | ${OBJCOPY} | ${CROSS_COMPILE}objcopy
  215. objdump | ${OBJDUMP} | ${CROSS_COMPILE}objdump
  216. dtc | ${DTC} | (no CROSS_COMPILE version)
  217. Args:
  218. name: Command name to run
  219. Returns:
  220. target_name: Exact command name to run instead
  221. extra_args: List of extra arguments to pass
  222. """
  223. env = dict(os.environ)
  224. target_name = None
  225. extra_args = []
  226. if name in ('as', 'ld', 'cc', 'cpp', 'ar', 'nm', 'ldr', 'strip',
  227. 'objcopy', 'objdump', 'dtc'):
  228. target_name, *extra_args = env.get(name.upper(), '').split(' ')
  229. elif name == 'c++':
  230. target_name, *extra_args = env.get('CXX', '').split(' ')
  231. if target_name:
  232. return target_name, extra_args
  233. if cross_compile is None:
  234. cross_compile = env.get('CROSS_COMPILE', '')
  235. if name in ('as', 'ar', 'nm', 'ldr', 'strip', 'objcopy', 'objdump'):
  236. target_name = cross_compile + name
  237. elif name == 'ld':
  238. try:
  239. if Run(cross_compile + 'ld.bfd', '-v'):
  240. target_name = cross_compile + 'ld.bfd'
  241. except:
  242. target_name = cross_compile + 'ld'
  243. elif name == 'cc':
  244. target_name = cross_compile + 'gcc'
  245. elif name == 'cpp':
  246. target_name = cross_compile + 'gcc'
  247. extra_args = ['-E']
  248. elif name == 'c++':
  249. target_name = cross_compile + 'g++'
  250. else:
  251. target_name = name
  252. return target_name, extra_args
  253. def Run(name, *args, **kwargs):
  254. """Run a tool with some arguments
  255. This runs a 'tool', which is a program used by binman to process files and
  256. perhaps produce some output. Tools can be located on the PATH or in a
  257. search path.
  258. Args:
  259. name: Command name to run
  260. args: Arguments to the tool
  261. for_host: True to resolve the command to the version for the host
  262. for_target: False to run the command as-is, without resolving it
  263. to the version for the compile target
  264. Returns:
  265. CommandResult object
  266. """
  267. try:
  268. binary = kwargs.get('binary')
  269. for_host = kwargs.get('for_host', False)
  270. for_target = kwargs.get('for_target', not for_host)
  271. env = None
  272. if tool_search_paths:
  273. env = dict(os.environ)
  274. env['PATH'] = ':'.join(tool_search_paths) + ':' + env['PATH']
  275. if for_target:
  276. name, extra_args = GetTargetCompileTool(name)
  277. args = tuple(extra_args) + args
  278. elif for_host:
  279. name, extra_args = GetHostCompileTool(name)
  280. args = tuple(extra_args) + args
  281. name = os.path.expanduser(name) # Expand paths containing ~
  282. all_args = (name,) + args
  283. result = command.RunPipe([all_args], capture=True, capture_stderr=True,
  284. env=env, raise_on_error=False, binary=binary)
  285. if result.return_code:
  286. raise Exception("Error %d running '%s': %s" %
  287. (result.return_code,' '.join(all_args),
  288. result.stderr))
  289. return result.stdout
  290. except:
  291. if env and not PathHasFile(env['PATH'], name):
  292. msg = "Please install tool '%s'" % name
  293. package = packages.get(name)
  294. if package:
  295. msg += " (e.g. from package '%s')" % package
  296. raise ValueError(msg)
  297. raise
  298. def Filename(fname):
  299. """Resolve a file path to an absolute path.
  300. If fname starts with ##/ and chroot is available, ##/ gets replaced with
  301. the chroot path. If chroot is not available, this file name can not be
  302. resolved, `None' is returned.
  303. If fname is not prepended with the above prefix, and is not an existing
  304. file, the actual file name is retrieved from the passed in string and the
  305. search_paths directories (if any) are searched to for the file. If found -
  306. the path to the found file is returned, `None' is returned otherwise.
  307. Args:
  308. fname: a string, the path to resolve.
  309. Returns:
  310. Absolute path to the file or None if not found.
  311. """
  312. if fname.startswith('##/'):
  313. if chroot_path:
  314. fname = os.path.join(chroot_path, fname[3:])
  315. else:
  316. return None
  317. # Search for a pathname that exists, and return it if found
  318. if fname and not os.path.exists(fname):
  319. for path in search_paths:
  320. pathname = os.path.join(path, os.path.basename(fname))
  321. if os.path.exists(pathname):
  322. return pathname
  323. # If not found, just return the standard, unchanged path
  324. return fname
  325. def ReadFile(fname, binary=True):
  326. """Read and return the contents of a file.
  327. Args:
  328. fname: path to filename to read, where ## signifiies the chroot.
  329. Returns:
  330. data read from file, as a string.
  331. """
  332. with open(Filename(fname), binary and 'rb' or 'r') as fd:
  333. data = fd.read()
  334. #self._out.Info("Read file '%s' size %d (%#0x)" %
  335. #(fname, len(data), len(data)))
  336. return data
  337. def WriteFile(fname, data, binary=True):
  338. """Write data into a file.
  339. Args:
  340. fname: path to filename to write
  341. data: data to write to file, as a string
  342. """
  343. #self._out.Info("Write file '%s' size %d (%#0x)" %
  344. #(fname, len(data), len(data)))
  345. with open(Filename(fname), binary and 'wb' or 'w') as fd:
  346. fd.write(data)
  347. def GetBytes(byte, size):
  348. """Get a string of bytes of a given size
  349. Args:
  350. byte: Numeric byte value to use
  351. size: Size of bytes/string to return
  352. Returns:
  353. A bytes type with 'byte' repeated 'size' times
  354. """
  355. return bytes([byte]) * size
  356. def ToBytes(string):
  357. """Convert a str type into a bytes type
  358. Args:
  359. string: string to convert
  360. Returns:
  361. A bytes type
  362. """
  363. return string.encode('utf-8')
  364. def ToString(bval):
  365. """Convert a bytes type into a str type
  366. Args:
  367. bval: bytes value to convert
  368. Returns:
  369. Python 3: A bytes type
  370. Python 2: A string type
  371. """
  372. return bval.decode('utf-8')
  373. def Compress(indata, algo, with_header=True):
  374. """Compress some data using a given algorithm
  375. Note that for lzma this uses an old version of the algorithm, not that
  376. provided by xz.
  377. This requires 'lz4' and 'lzma_alone' tools. It also requires an output
  378. directory to be previously set up, by calling PrepareOutputDir().
  379. Care is taken to use unique temporary files so that this function can be
  380. called from multiple threads.
  381. Args:
  382. indata: Input data to compress
  383. algo: Algorithm to use ('none', 'gzip', 'lz4' or 'lzma')
  384. Returns:
  385. Compressed data
  386. """
  387. if algo == 'none':
  388. return indata
  389. fname = tempfile.NamedTemporaryFile(prefix='%s.comp.tmp' % algo,
  390. dir=outdir).name
  391. WriteFile(fname, indata)
  392. if algo == 'lz4':
  393. data = Run('lz4', '--no-frame-crc', '-B4', '-5', '-c', fname,
  394. binary=True)
  395. # cbfstool uses a very old version of lzma
  396. elif algo == 'lzma':
  397. outfname = tempfile.NamedTemporaryFile(prefix='%s.comp.otmp' % algo,
  398. dir=outdir).name
  399. Run('lzma_alone', 'e', fname, outfname, '-lc1', '-lp0', '-pb0', '-d8')
  400. data = ReadFile(outfname)
  401. elif algo == 'gzip':
  402. data = Run('gzip', '-c', fname, binary=True)
  403. else:
  404. raise ValueError("Unknown algorithm '%s'" % algo)
  405. if with_header:
  406. hdr = struct.pack('<I', len(data))
  407. data = hdr + data
  408. return data
  409. def Decompress(indata, algo, with_header=True):
  410. """Decompress some data using a given algorithm
  411. Note that for lzma this uses an old version of the algorithm, not that
  412. provided by xz.
  413. This requires 'lz4' and 'lzma_alone' tools. It also requires an output
  414. directory to be previously set up, by calling PrepareOutputDir().
  415. Args:
  416. indata: Input data to decompress
  417. algo: Algorithm to use ('none', 'gzip', 'lz4' or 'lzma')
  418. Returns:
  419. Compressed data
  420. """
  421. if algo == 'none':
  422. return indata
  423. if with_header:
  424. data_len = struct.unpack('<I', indata[:4])[0]
  425. indata = indata[4:4 + data_len]
  426. fname = GetOutputFilename('%s.decomp.tmp' % algo)
  427. with open(fname, 'wb') as fd:
  428. fd.write(indata)
  429. if algo == 'lz4':
  430. data = Run('lz4', '-dc', fname, binary=True)
  431. elif algo == 'lzma':
  432. outfname = GetOutputFilename('%s.decomp.otmp' % algo)
  433. Run('lzma_alone', 'd', fname, outfname)
  434. data = ReadFile(outfname, binary=True)
  435. elif algo == 'gzip':
  436. data = Run('gzip', '-cd', fname, binary=True)
  437. else:
  438. raise ValueError("Unknown algorithm '%s'" % algo)
  439. return data
  440. CMD_CREATE, CMD_DELETE, CMD_ADD, CMD_REPLACE, CMD_EXTRACT = range(5)
  441. IFWITOOL_CMDS = {
  442. CMD_CREATE: 'create',
  443. CMD_DELETE: 'delete',
  444. CMD_ADD: 'add',
  445. CMD_REPLACE: 'replace',
  446. CMD_EXTRACT: 'extract',
  447. }
  448. def RunIfwiTool(ifwi_file, cmd, fname=None, subpart=None, entry_name=None):
  449. """Run ifwitool with the given arguments:
  450. Args:
  451. ifwi_file: IFWI file to operation on
  452. cmd: Command to execute (CMD_...)
  453. fname: Filename of file to add/replace/extract/create (None for
  454. CMD_DELETE)
  455. subpart: Name of sub-partition to operation on (None for CMD_CREATE)
  456. entry_name: Name of directory entry to operate on, or None if none
  457. """
  458. args = ['ifwitool', ifwi_file]
  459. args.append(IFWITOOL_CMDS[cmd])
  460. if fname:
  461. args += ['-f', fname]
  462. if subpart:
  463. args += ['-n', subpart]
  464. if entry_name:
  465. args += ['-d', '-e', entry_name]
  466. Run(*args)
  467. def ToHex(val):
  468. """Convert an integer value (or None) to a string
  469. Returns:
  470. hex value, or 'None' if the value is None
  471. """
  472. return 'None' if val is None else '%#x' % val
  473. def ToHexSize(val):
  474. """Return the size of an object in hex
  475. Returns:
  476. hex value of size, or 'None' if the value is None
  477. """
  478. return 'None' if val is None else '%#x' % len(val)