tools.py 18 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597
  1. # SPDX-License-Identifier: GPL-2.0+
  2. #
  3. # Copyright (c) 2016 Google, Inc
  4. #
  5. import glob
  6. import os
  7. import shutil
  8. import struct
  9. import sys
  10. import tempfile
  11. from patman import command
  12. from patman import tout
  13. # Output directly (generally this is temporary)
  14. outdir = None
  15. # True to keep the output directory around after exiting
  16. preserve_outdir = False
  17. # Path to the Chrome OS chroot, if we know it
  18. chroot_path = None
  19. # Search paths to use for Filename(), used to find files
  20. search_paths = []
  21. tool_search_paths = []
  22. # Tools and the packages that contain them, on debian
  23. packages = {
  24. 'lz4': 'liblz4-tool',
  25. }
  26. # List of paths to use when looking for an input file
  27. indir = []
  28. def PrepareOutputDir(dirname, preserve=False):
  29. """Select an output directory, ensuring it exists.
  30. This either creates a temporary directory or checks that the one supplied
  31. by the user is valid. For a temporary directory, it makes a note to
  32. remove it later if required.
  33. Args:
  34. dirname: a string, name of the output directory to use to store
  35. intermediate and output files. If is None - create a temporary
  36. directory.
  37. preserve: a Boolean. If outdir above is None and preserve is False, the
  38. created temporary directory will be destroyed on exit.
  39. Raises:
  40. OSError: If it cannot create the output directory.
  41. """
  42. global outdir, preserve_outdir
  43. preserve_outdir = dirname or preserve
  44. if dirname:
  45. outdir = dirname
  46. if not os.path.isdir(outdir):
  47. try:
  48. os.makedirs(outdir)
  49. except OSError as err:
  50. raise CmdError("Cannot make output directory '%s': '%s'" %
  51. (outdir, err.strerror))
  52. tout.Debug("Using output directory '%s'" % outdir)
  53. else:
  54. outdir = tempfile.mkdtemp(prefix='binman.')
  55. tout.Debug("Using temporary directory '%s'" % outdir)
  56. def _RemoveOutputDir():
  57. global outdir
  58. shutil.rmtree(outdir)
  59. tout.Debug("Deleted temporary directory '%s'" % outdir)
  60. outdir = None
  61. def FinaliseOutputDir():
  62. global outdir, preserve_outdir
  63. """Tidy up: delete output directory if temporary and not preserved."""
  64. if outdir and not preserve_outdir:
  65. _RemoveOutputDir()
  66. outdir = None
  67. def GetOutputFilename(fname):
  68. """Return a filename within the output directory.
  69. Args:
  70. fname: Filename to use for new file
  71. Returns:
  72. The full path of the filename, within the output directory
  73. """
  74. return os.path.join(outdir, fname)
  75. def _FinaliseForTest():
  76. """Remove the output directory (for use by tests)"""
  77. global outdir
  78. if outdir:
  79. _RemoveOutputDir()
  80. outdir = None
  81. def SetInputDirs(dirname):
  82. """Add a list of input directories, where input files are kept.
  83. Args:
  84. dirname: a list of paths to input directories to use for obtaining
  85. files needed by binman to place in the image.
  86. """
  87. global indir
  88. indir = dirname
  89. tout.Debug("Using input directories %s" % indir)
  90. def GetInputFilename(fname, allow_missing=False):
  91. """Return a filename for use as input.
  92. Args:
  93. fname: Filename to use for new file
  94. allow_missing: True if the filename can be missing
  95. Returns:
  96. The full path of the filename, within the input directory, or
  97. None on error
  98. """
  99. if not indir or fname[:1] == '/':
  100. return fname
  101. for dirname in indir:
  102. pathname = os.path.join(dirname, fname)
  103. if os.path.exists(pathname):
  104. return pathname
  105. if allow_missing:
  106. return None
  107. raise ValueError("Filename '%s' not found in input path (%s) (cwd='%s')" %
  108. (fname, ','.join(indir), os.getcwd()))
  109. def GetInputFilenameGlob(pattern):
  110. """Return a list of filenames for use as input.
  111. Args:
  112. pattern: Filename pattern to search for
  113. Returns:
  114. A list of matching files in all input directories
  115. """
  116. if not indir:
  117. return glob.glob(fname)
  118. files = []
  119. for dirname in indir:
  120. pathname = os.path.join(dirname, pattern)
  121. files += glob.glob(pathname)
  122. return sorted(files)
  123. def Align(pos, align):
  124. if align:
  125. mask = align - 1
  126. pos = (pos + mask) & ~mask
  127. return pos
  128. def NotPowerOfTwo(num):
  129. return num and (num & (num - 1))
  130. def SetToolPaths(toolpaths):
  131. """Set the path to search for tools
  132. Args:
  133. toolpaths: List of paths to search for tools executed by Run()
  134. """
  135. global tool_search_paths
  136. tool_search_paths = toolpaths
  137. def PathHasFile(path_spec, fname):
  138. """Check if a given filename is in the PATH
  139. Args:
  140. path_spec: Value of PATH variable to check
  141. fname: Filename to check
  142. Returns:
  143. True if found, False if not
  144. """
  145. for dir in path_spec.split(':'):
  146. if os.path.exists(os.path.join(dir, fname)):
  147. return True
  148. return False
  149. def GetTargetCompileTool(name, cross_compile=None):
  150. """Get the target-specific version for a compile tool
  151. This first checks the environment variables that specify which
  152. version of the tool should be used (e.g. ${CC}). If those aren't
  153. specified, it checks the CROSS_COMPILE variable as a prefix for the
  154. tool with some substitutions (e.g. "${CROSS_COMPILE}gcc" for cc).
  155. The following table lists the target-specific versions of the tools
  156. this function resolves to:
  157. Compile Tool | First choice | Second choice
  158. --------------+----------------+----------------------------
  159. as | ${AS} | ${CROSS_COMPILE}as
  160. ld | ${LD} | ${CROSS_COMPILE}ld.bfd
  161. | | or ${CROSS_COMPILE}ld
  162. cc | ${CC} | ${CROSS_COMPILE}gcc
  163. cpp | ${CPP} | ${CROSS_COMPILE}gcc -E
  164. c++ | ${CXX} | ${CROSS_COMPILE}g++
  165. ar | ${AR} | ${CROSS_COMPILE}ar
  166. nm | ${NM} | ${CROSS_COMPILE}nm
  167. ldr | ${LDR} | ${CROSS_COMPILE}ldr
  168. strip | ${STRIP} | ${CROSS_COMPILE}strip
  169. objcopy | ${OBJCOPY} | ${CROSS_COMPILE}objcopy
  170. objdump | ${OBJDUMP} | ${CROSS_COMPILE}objdump
  171. dtc | ${DTC} | (no CROSS_COMPILE version)
  172. Args:
  173. name: Command name to run
  174. Returns:
  175. target_name: Exact command name to run instead
  176. extra_args: List of extra arguments to pass
  177. """
  178. env = dict(os.environ)
  179. target_name = None
  180. extra_args = []
  181. if name in ('as', 'ld', 'cc', 'cpp', 'ar', 'nm', 'ldr', 'strip',
  182. 'objcopy', 'objdump', 'dtc'):
  183. target_name, *extra_args = env.get(name.upper(), '').split(' ')
  184. elif name == 'c++':
  185. target_name, *extra_args = env.get('CXX', '').split(' ')
  186. if target_name:
  187. return target_name, extra_args
  188. if cross_compile is None:
  189. cross_compile = env.get('CROSS_COMPILE', '')
  190. if not cross_compile:
  191. return name, []
  192. if name in ('as', 'ar', 'nm', 'ldr', 'strip', 'objcopy', 'objdump'):
  193. target_name = cross_compile + name
  194. elif name == 'ld':
  195. try:
  196. if Run(cross_compile + 'ld.bfd', '-v'):
  197. target_name = cross_compile + 'ld.bfd'
  198. except:
  199. target_name = cross_compile + 'ld'
  200. elif name == 'cc':
  201. target_name = cross_compile + 'gcc'
  202. elif name == 'cpp':
  203. target_name = cross_compile + 'gcc'
  204. extra_args = ['-E']
  205. elif name == 'c++':
  206. target_name = cross_compile + 'g++'
  207. else:
  208. target_name = name
  209. return target_name, extra_args
  210. def Run(name, *args, **kwargs):
  211. """Run a tool with some arguments
  212. This runs a 'tool', which is a program used by binman to process files and
  213. perhaps produce some output. Tools can be located on the PATH or in a
  214. search path.
  215. Args:
  216. name: Command name to run
  217. args: Arguments to the tool
  218. for_target: False to run the command as-is, without resolving it
  219. to the version for the compile target
  220. Returns:
  221. CommandResult object
  222. """
  223. try:
  224. binary = kwargs.get('binary')
  225. for_target = kwargs.get('for_target', True)
  226. env = None
  227. if tool_search_paths:
  228. env = dict(os.environ)
  229. env['PATH'] = ':'.join(tool_search_paths) + ':' + env['PATH']
  230. if for_target:
  231. name, extra_args = GetTargetCompileTool(name)
  232. args = tuple(extra_args) + args
  233. all_args = (name,) + args
  234. result = command.RunPipe([all_args], capture=True, capture_stderr=True,
  235. env=env, raise_on_error=False, binary=binary)
  236. if result.return_code:
  237. raise Exception("Error %d running '%s': %s" %
  238. (result.return_code,' '.join(all_args),
  239. result.stderr))
  240. return result.stdout
  241. except:
  242. if env and not PathHasFile(env['PATH'], name):
  243. msg = "Please install tool '%s'" % name
  244. package = packages.get(name)
  245. if package:
  246. msg += " (e.g. from package '%s')" % package
  247. raise ValueError(msg)
  248. raise
  249. def Filename(fname):
  250. """Resolve a file path to an absolute path.
  251. If fname starts with ##/ and chroot is available, ##/ gets replaced with
  252. the chroot path. If chroot is not available, this file name can not be
  253. resolved, `None' is returned.
  254. If fname is not prepended with the above prefix, and is not an existing
  255. file, the actual file name is retrieved from the passed in string and the
  256. search_paths directories (if any) are searched to for the file. If found -
  257. the path to the found file is returned, `None' is returned otherwise.
  258. Args:
  259. fname: a string, the path to resolve.
  260. Returns:
  261. Absolute path to the file or None if not found.
  262. """
  263. if fname.startswith('##/'):
  264. if chroot_path:
  265. fname = os.path.join(chroot_path, fname[3:])
  266. else:
  267. return None
  268. # Search for a pathname that exists, and return it if found
  269. if fname and not os.path.exists(fname):
  270. for path in search_paths:
  271. pathname = os.path.join(path, os.path.basename(fname))
  272. if os.path.exists(pathname):
  273. return pathname
  274. # If not found, just return the standard, unchanged path
  275. return fname
  276. def ReadFile(fname, binary=True):
  277. """Read and return the contents of a file.
  278. Args:
  279. fname: path to filename to read, where ## signifiies the chroot.
  280. Returns:
  281. data read from file, as a string.
  282. """
  283. with open(Filename(fname), binary and 'rb' or 'r') as fd:
  284. data = fd.read()
  285. #self._out.Info("Read file '%s' size %d (%#0x)" %
  286. #(fname, len(data), len(data)))
  287. return data
  288. def WriteFile(fname, data, binary=True):
  289. """Write data into a file.
  290. Args:
  291. fname: path to filename to write
  292. data: data to write to file, as a string
  293. """
  294. #self._out.Info("Write file '%s' size %d (%#0x)" %
  295. #(fname, len(data), len(data)))
  296. with open(Filename(fname), binary and 'wb' or 'w') as fd:
  297. fd.write(data)
  298. def GetBytes(byte, size):
  299. """Get a string of bytes of a given size
  300. This handles the unfortunate different between Python 2 and Python 2.
  301. Args:
  302. byte: Numeric byte value to use
  303. size: Size of bytes/string to return
  304. Returns:
  305. A bytes type with 'byte' repeated 'size' times
  306. """
  307. if sys.version_info[0] >= 3:
  308. data = bytes([byte]) * size
  309. else:
  310. data = chr(byte) * size
  311. return data
  312. def ToUnicode(val):
  313. """Make sure a value is a unicode string
  314. This allows some amount of compatibility between Python 2 and Python3. For
  315. the former, it returns a unicode object.
  316. Args:
  317. val: string or unicode object
  318. Returns:
  319. unicode version of val
  320. """
  321. if sys.version_info[0] >= 3:
  322. return val
  323. return val if isinstance(val, unicode) else val.decode('utf-8')
  324. def FromUnicode(val):
  325. """Make sure a value is a non-unicode string
  326. This allows some amount of compatibility between Python 2 and Python3. For
  327. the former, it converts a unicode object to a string.
  328. Args:
  329. val: string or unicode object
  330. Returns:
  331. non-unicode version of val
  332. """
  333. if sys.version_info[0] >= 3:
  334. return val
  335. return val if isinstance(val, str) else val.encode('utf-8')
  336. def ToByte(ch):
  337. """Convert a character to an ASCII value
  338. This is useful because in Python 2 bytes is an alias for str, but in
  339. Python 3 they are separate types. This function converts the argument to
  340. an ASCII value in either case.
  341. Args:
  342. ch: A string (Python 2) or byte (Python 3) value
  343. Returns:
  344. integer ASCII value for ch
  345. """
  346. return ord(ch) if type(ch) == str else ch
  347. def ToChar(byte):
  348. """Convert a byte to a character
  349. This is useful because in Python 2 bytes is an alias for str, but in
  350. Python 3 they are separate types. This function converts an ASCII value to
  351. a value with the appropriate type in either case.
  352. Args:
  353. byte: A byte or str value
  354. """
  355. return chr(byte) if type(byte) != str else byte
  356. def ToChars(byte_list):
  357. """Convert a list of bytes to a str/bytes type
  358. Args:
  359. byte_list: List of ASCII values representing the string
  360. Returns:
  361. string made by concatenating all the ASCII values
  362. """
  363. return ''.join([chr(byte) for byte in byte_list])
  364. def ToBytes(string):
  365. """Convert a str type into a bytes type
  366. Args:
  367. string: string to convert
  368. Returns:
  369. Python 3: A bytes type
  370. Python 2: A string type
  371. """
  372. if sys.version_info[0] >= 3:
  373. return string.encode('utf-8')
  374. return string
  375. def ToString(bval):
  376. """Convert a bytes type into a str type
  377. Args:
  378. bval: bytes value to convert
  379. Returns:
  380. Python 3: A bytes type
  381. Python 2: A string type
  382. """
  383. return bval.decode('utf-8')
  384. def Compress(indata, algo, with_header=True):
  385. """Compress some data using a given algorithm
  386. Note that for lzma this uses an old version of the algorithm, not that
  387. provided by xz.
  388. This requires 'lz4' and 'lzma_alone' tools. It also requires an output
  389. directory to be previously set up, by calling PrepareOutputDir().
  390. Args:
  391. indata: Input data to compress
  392. algo: Algorithm to use ('none', 'gzip', 'lz4' or 'lzma')
  393. Returns:
  394. Compressed data
  395. """
  396. if algo == 'none':
  397. return indata
  398. fname = GetOutputFilename('%s.comp.tmp' % algo)
  399. WriteFile(fname, indata)
  400. if algo == 'lz4':
  401. data = Run('lz4', '--no-frame-crc', '-c', fname, binary=True)
  402. # cbfstool uses a very old version of lzma
  403. elif algo == 'lzma':
  404. outfname = GetOutputFilename('%s.comp.otmp' % algo)
  405. Run('lzma_alone', 'e', fname, outfname, '-lc1', '-lp0', '-pb0', '-d8')
  406. data = ReadFile(outfname)
  407. elif algo == 'gzip':
  408. data = Run('gzip', '-c', fname, binary=True)
  409. else:
  410. raise ValueError("Unknown algorithm '%s'" % algo)
  411. if with_header:
  412. hdr = struct.pack('<I', len(data))
  413. data = hdr + data
  414. return data
  415. def Decompress(indata, algo, with_header=True):
  416. """Decompress some data using a given algorithm
  417. Note that for lzma this uses an old version of the algorithm, not that
  418. provided by xz.
  419. This requires 'lz4' and 'lzma_alone' tools. It also requires an output
  420. directory to be previously set up, by calling PrepareOutputDir().
  421. Args:
  422. indata: Input data to decompress
  423. algo: Algorithm to use ('none', 'gzip', 'lz4' or 'lzma')
  424. Returns:
  425. Compressed data
  426. """
  427. if algo == 'none':
  428. return indata
  429. if with_header:
  430. data_len = struct.unpack('<I', indata[:4])[0]
  431. indata = indata[4:4 + data_len]
  432. fname = GetOutputFilename('%s.decomp.tmp' % algo)
  433. with open(fname, 'wb') as fd:
  434. fd.write(indata)
  435. if algo == 'lz4':
  436. data = Run('lz4', '-dc', fname, binary=True)
  437. elif algo == 'lzma':
  438. outfname = GetOutputFilename('%s.decomp.otmp' % algo)
  439. Run('lzma_alone', 'd', fname, outfname)
  440. data = ReadFile(outfname, binary=True)
  441. elif algo == 'gzip':
  442. data = Run('gzip', '-cd', fname, binary=True)
  443. else:
  444. raise ValueError("Unknown algorithm '%s'" % algo)
  445. return data
  446. CMD_CREATE, CMD_DELETE, CMD_ADD, CMD_REPLACE, CMD_EXTRACT = range(5)
  447. IFWITOOL_CMDS = {
  448. CMD_CREATE: 'create',
  449. CMD_DELETE: 'delete',
  450. CMD_ADD: 'add',
  451. CMD_REPLACE: 'replace',
  452. CMD_EXTRACT: 'extract',
  453. }
  454. def RunIfwiTool(ifwi_file, cmd, fname=None, subpart=None, entry_name=None):
  455. """Run ifwitool with the given arguments:
  456. Args:
  457. ifwi_file: IFWI file to operation on
  458. cmd: Command to execute (CMD_...)
  459. fname: Filename of file to add/replace/extract/create (None for
  460. CMD_DELETE)
  461. subpart: Name of sub-partition to operation on (None for CMD_CREATE)
  462. entry_name: Name of directory entry to operate on, or None if none
  463. """
  464. args = ['ifwitool', ifwi_file]
  465. args.append(IFWITOOL_CMDS[cmd])
  466. if fname:
  467. args += ['-f', fname]
  468. if subpart:
  469. args += ['-n', subpart]
  470. if entry_name:
  471. args += ['-d', '-e', entry_name]
  472. Run(*args)
  473. def ToHex(val):
  474. """Convert an integer value (or None) to a string
  475. Returns:
  476. hex value, or 'None' if the value is None
  477. """
  478. return 'None' if val is None else '%#x' % val
  479. def ToHexSize(val):
  480. """Return the size of an object in hex
  481. Returns:
  482. hex value of size, or 'None' if the value is None
  483. """
  484. return 'None' if val is None else '%#x' % len(val)