tools.py 14 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516
  1. # SPDX-License-Identifier: GPL-2.0+
  2. #
  3. # Copyright (c) 2016 Google, Inc
  4. #
  5. import glob
  6. import os
  7. import shutil
  8. import struct
  9. import sys
  10. import tempfile
  11. from patman import command
  12. from patman import tout
  13. # Output directly (generally this is temporary)
  14. outdir = None
  15. # True to keep the output directory around after exiting
  16. preserve_outdir = False
  17. # Path to the Chrome OS chroot, if we know it
  18. chroot_path = None
  19. # Search paths to use for Filename(), used to find files
  20. search_paths = []
  21. tool_search_paths = []
  22. # Tools and the packages that contain them, on debian
  23. packages = {
  24. 'lz4': 'liblz4-tool',
  25. }
  26. # List of paths to use when looking for an input file
  27. indir = []
  28. def PrepareOutputDir(dirname, preserve=False):
  29. """Select an output directory, ensuring it exists.
  30. This either creates a temporary directory or checks that the one supplied
  31. by the user is valid. For a temporary directory, it makes a note to
  32. remove it later if required.
  33. Args:
  34. dirname: a string, name of the output directory to use to store
  35. intermediate and output files. If is None - create a temporary
  36. directory.
  37. preserve: a Boolean. If outdir above is None and preserve is False, the
  38. created temporary directory will be destroyed on exit.
  39. Raises:
  40. OSError: If it cannot create the output directory.
  41. """
  42. global outdir, preserve_outdir
  43. preserve_outdir = dirname or preserve
  44. if dirname:
  45. outdir = dirname
  46. if not os.path.isdir(outdir):
  47. try:
  48. os.makedirs(outdir)
  49. except OSError as err:
  50. raise CmdError("Cannot make output directory '%s': '%s'" %
  51. (outdir, err.strerror))
  52. tout.Debug("Using output directory '%s'" % outdir)
  53. else:
  54. outdir = tempfile.mkdtemp(prefix='binman.')
  55. tout.Debug("Using temporary directory '%s'" % outdir)
  56. def _RemoveOutputDir():
  57. global outdir
  58. shutil.rmtree(outdir)
  59. tout.Debug("Deleted temporary directory '%s'" % outdir)
  60. outdir = None
  61. def FinaliseOutputDir():
  62. global outdir, preserve_outdir
  63. """Tidy up: delete output directory if temporary and not preserved."""
  64. if outdir and not preserve_outdir:
  65. _RemoveOutputDir()
  66. outdir = None
  67. def GetOutputFilename(fname):
  68. """Return a filename within the output directory.
  69. Args:
  70. fname: Filename to use for new file
  71. Returns:
  72. The full path of the filename, within the output directory
  73. """
  74. return os.path.join(outdir, fname)
  75. def _FinaliseForTest():
  76. """Remove the output directory (for use by tests)"""
  77. global outdir
  78. if outdir:
  79. _RemoveOutputDir()
  80. outdir = None
  81. def SetInputDirs(dirname):
  82. """Add a list of input directories, where input files are kept.
  83. Args:
  84. dirname: a list of paths to input directories to use for obtaining
  85. files needed by binman to place in the image.
  86. """
  87. global indir
  88. indir = dirname
  89. tout.Debug("Using input directories %s" % indir)
  90. def GetInputFilename(fname):
  91. """Return a filename for use as input.
  92. Args:
  93. fname: Filename to use for new file
  94. Returns:
  95. The full path of the filename, within the input directory
  96. """
  97. if not indir or fname[:1] == '/':
  98. return fname
  99. for dirname in indir:
  100. pathname = os.path.join(dirname, fname)
  101. if os.path.exists(pathname):
  102. return pathname
  103. raise ValueError("Filename '%s' not found in input path (%s) (cwd='%s')" %
  104. (fname, ','.join(indir), os.getcwd()))
  105. def GetInputFilenameGlob(pattern):
  106. """Return a list of filenames for use as input.
  107. Args:
  108. pattern: Filename pattern to search for
  109. Returns:
  110. A list of matching files in all input directories
  111. """
  112. if not indir:
  113. return glob.glob(fname)
  114. files = []
  115. for dirname in indir:
  116. pathname = os.path.join(dirname, pattern)
  117. files += glob.glob(pathname)
  118. return sorted(files)
  119. def Align(pos, align):
  120. if align:
  121. mask = align - 1
  122. pos = (pos + mask) & ~mask
  123. return pos
  124. def NotPowerOfTwo(num):
  125. return num and (num & (num - 1))
  126. def SetToolPaths(toolpaths):
  127. """Set the path to search for tools
  128. Args:
  129. toolpaths: List of paths to search for tools executed by Run()
  130. """
  131. global tool_search_paths
  132. tool_search_paths = toolpaths
  133. def PathHasFile(path_spec, fname):
  134. """Check if a given filename is in the PATH
  135. Args:
  136. path_spec: Value of PATH variable to check
  137. fname: Filename to check
  138. Returns:
  139. True if found, False if not
  140. """
  141. for dir in path_spec.split(':'):
  142. if os.path.exists(os.path.join(dir, fname)):
  143. return True
  144. return False
  145. def Run(name, *args, **kwargs):
  146. """Run a tool with some arguments
  147. This runs a 'tool', which is a program used by binman to process files and
  148. perhaps produce some output. Tools can be located on the PATH or in a
  149. search path.
  150. Args:
  151. name: Command name to run
  152. args: Arguments to the tool
  153. Returns:
  154. CommandResult object
  155. """
  156. try:
  157. binary = kwargs.get('binary')
  158. env = None
  159. if tool_search_paths:
  160. env = dict(os.environ)
  161. env['PATH'] = ':'.join(tool_search_paths) + ':' + env['PATH']
  162. all_args = (name,) + args
  163. result = command.RunPipe([all_args], capture=True, capture_stderr=True,
  164. env=env, raise_on_error=False, binary=binary)
  165. if result.return_code:
  166. raise Exception("Error %d running '%s': %s" %
  167. (result.return_code,' '.join(all_args),
  168. result.stderr))
  169. return result.stdout
  170. except:
  171. if env and not PathHasFile(env['PATH'], name):
  172. msg = "Please install tool '%s'" % name
  173. package = packages.get(name)
  174. if package:
  175. msg += " (e.g. from package '%s')" % package
  176. raise ValueError(msg)
  177. raise
  178. def Filename(fname):
  179. """Resolve a file path to an absolute path.
  180. If fname starts with ##/ and chroot is available, ##/ gets replaced with
  181. the chroot path. If chroot is not available, this file name can not be
  182. resolved, `None' is returned.
  183. If fname is not prepended with the above prefix, and is not an existing
  184. file, the actual file name is retrieved from the passed in string and the
  185. search_paths directories (if any) are searched to for the file. If found -
  186. the path to the found file is returned, `None' is returned otherwise.
  187. Args:
  188. fname: a string, the path to resolve.
  189. Returns:
  190. Absolute path to the file or None if not found.
  191. """
  192. if fname.startswith('##/'):
  193. if chroot_path:
  194. fname = os.path.join(chroot_path, fname[3:])
  195. else:
  196. return None
  197. # Search for a pathname that exists, and return it if found
  198. if fname and not os.path.exists(fname):
  199. for path in search_paths:
  200. pathname = os.path.join(path, os.path.basename(fname))
  201. if os.path.exists(pathname):
  202. return pathname
  203. # If not found, just return the standard, unchanged path
  204. return fname
  205. def ReadFile(fname, binary=True):
  206. """Read and return the contents of a file.
  207. Args:
  208. fname: path to filename to read, where ## signifiies the chroot.
  209. Returns:
  210. data read from file, as a string.
  211. """
  212. with open(Filename(fname), binary and 'rb' or 'r') as fd:
  213. data = fd.read()
  214. #self._out.Info("Read file '%s' size %d (%#0x)" %
  215. #(fname, len(data), len(data)))
  216. return data
  217. def WriteFile(fname, data):
  218. """Write data into a file.
  219. Args:
  220. fname: path to filename to write
  221. data: data to write to file, as a string
  222. """
  223. #self._out.Info("Write file '%s' size %d (%#0x)" %
  224. #(fname, len(data), len(data)))
  225. with open(Filename(fname), 'wb') as fd:
  226. fd.write(data)
  227. def GetBytes(byte, size):
  228. """Get a string of bytes of a given size
  229. This handles the unfortunate different between Python 2 and Python 2.
  230. Args:
  231. byte: Numeric byte value to use
  232. size: Size of bytes/string to return
  233. Returns:
  234. A bytes type with 'byte' repeated 'size' times
  235. """
  236. if sys.version_info[0] >= 3:
  237. data = bytes([byte]) * size
  238. else:
  239. data = chr(byte) * size
  240. return data
  241. def ToUnicode(val):
  242. """Make sure a value is a unicode string
  243. This allows some amount of compatibility between Python 2 and Python3. For
  244. the former, it returns a unicode object.
  245. Args:
  246. val: string or unicode object
  247. Returns:
  248. unicode version of val
  249. """
  250. if sys.version_info[0] >= 3:
  251. return val
  252. return val if isinstance(val, unicode) else val.decode('utf-8')
  253. def FromUnicode(val):
  254. """Make sure a value is a non-unicode string
  255. This allows some amount of compatibility between Python 2 and Python3. For
  256. the former, it converts a unicode object to a string.
  257. Args:
  258. val: string or unicode object
  259. Returns:
  260. non-unicode version of val
  261. """
  262. if sys.version_info[0] >= 3:
  263. return val
  264. return val if isinstance(val, str) else val.encode('utf-8')
  265. def ToByte(ch):
  266. """Convert a character to an ASCII value
  267. This is useful because in Python 2 bytes is an alias for str, but in
  268. Python 3 they are separate types. This function converts the argument to
  269. an ASCII value in either case.
  270. Args:
  271. ch: A string (Python 2) or byte (Python 3) value
  272. Returns:
  273. integer ASCII value for ch
  274. """
  275. return ord(ch) if type(ch) == str else ch
  276. def ToChar(byte):
  277. """Convert a byte to a character
  278. This is useful because in Python 2 bytes is an alias for str, but in
  279. Python 3 they are separate types. This function converts an ASCII value to
  280. a value with the appropriate type in either case.
  281. Args:
  282. byte: A byte or str value
  283. """
  284. return chr(byte) if type(byte) != str else byte
  285. def ToChars(byte_list):
  286. """Convert a list of bytes to a str/bytes type
  287. Args:
  288. byte_list: List of ASCII values representing the string
  289. Returns:
  290. string made by concatenating all the ASCII values
  291. """
  292. return ''.join([chr(byte) for byte in byte_list])
  293. def ToBytes(string):
  294. """Convert a str type into a bytes type
  295. Args:
  296. string: string to convert
  297. Returns:
  298. Python 3: A bytes type
  299. Python 2: A string type
  300. """
  301. if sys.version_info[0] >= 3:
  302. return string.encode('utf-8')
  303. return string
  304. def ToString(bval):
  305. """Convert a bytes type into a str type
  306. Args:
  307. bval: bytes value to convert
  308. Returns:
  309. Python 3: A bytes type
  310. Python 2: A string type
  311. """
  312. return bval.decode('utf-8')
  313. def Compress(indata, algo, with_header=True):
  314. """Compress some data using a given algorithm
  315. Note that for lzma this uses an old version of the algorithm, not that
  316. provided by xz.
  317. This requires 'lz4' and 'lzma_alone' tools. It also requires an output
  318. directory to be previously set up, by calling PrepareOutputDir().
  319. Args:
  320. indata: Input data to compress
  321. algo: Algorithm to use ('none', 'gzip', 'lz4' or 'lzma')
  322. Returns:
  323. Compressed data
  324. """
  325. if algo == 'none':
  326. return indata
  327. fname = GetOutputFilename('%s.comp.tmp' % algo)
  328. WriteFile(fname, indata)
  329. if algo == 'lz4':
  330. data = Run('lz4', '--no-frame-crc', '-c', fname, binary=True)
  331. # cbfstool uses a very old version of lzma
  332. elif algo == 'lzma':
  333. outfname = GetOutputFilename('%s.comp.otmp' % algo)
  334. Run('lzma_alone', 'e', fname, outfname, '-lc1', '-lp0', '-pb0', '-d8')
  335. data = ReadFile(outfname)
  336. elif algo == 'gzip':
  337. data = Run('gzip', '-c', fname, binary=True)
  338. else:
  339. raise ValueError("Unknown algorithm '%s'" % algo)
  340. if with_header:
  341. hdr = struct.pack('<I', len(data))
  342. data = hdr + data
  343. return data
  344. def Decompress(indata, algo, with_header=True):
  345. """Decompress some data using a given algorithm
  346. Note that for lzma this uses an old version of the algorithm, not that
  347. provided by xz.
  348. This requires 'lz4' and 'lzma_alone' tools. It also requires an output
  349. directory to be previously set up, by calling PrepareOutputDir().
  350. Args:
  351. indata: Input data to decompress
  352. algo: Algorithm to use ('none', 'gzip', 'lz4' or 'lzma')
  353. Returns:
  354. Compressed data
  355. """
  356. if algo == 'none':
  357. return indata
  358. if with_header:
  359. data_len = struct.unpack('<I', indata[:4])[0]
  360. indata = indata[4:4 + data_len]
  361. fname = GetOutputFilename('%s.decomp.tmp' % algo)
  362. with open(fname, 'wb') as fd:
  363. fd.write(indata)
  364. if algo == 'lz4':
  365. data = Run('lz4', '-dc', fname, binary=True)
  366. elif algo == 'lzma':
  367. outfname = GetOutputFilename('%s.decomp.otmp' % algo)
  368. Run('lzma_alone', 'd', fname, outfname)
  369. data = ReadFile(outfname, binary=True)
  370. elif algo == 'gzip':
  371. data = Run('gzip', '-cd', fname, binary=True)
  372. else:
  373. raise ValueError("Unknown algorithm '%s'" % algo)
  374. return data
  375. CMD_CREATE, CMD_DELETE, CMD_ADD, CMD_REPLACE, CMD_EXTRACT = range(5)
  376. IFWITOOL_CMDS = {
  377. CMD_CREATE: 'create',
  378. CMD_DELETE: 'delete',
  379. CMD_ADD: 'add',
  380. CMD_REPLACE: 'replace',
  381. CMD_EXTRACT: 'extract',
  382. }
  383. def RunIfwiTool(ifwi_file, cmd, fname=None, subpart=None, entry_name=None):
  384. """Run ifwitool with the given arguments:
  385. Args:
  386. ifwi_file: IFWI file to operation on
  387. cmd: Command to execute (CMD_...)
  388. fname: Filename of file to add/replace/extract/create (None for
  389. CMD_DELETE)
  390. subpart: Name of sub-partition to operation on (None for CMD_CREATE)
  391. entry_name: Name of directory entry to operate on, or None if none
  392. """
  393. args = ['ifwitool', ifwi_file]
  394. args.append(IFWITOOL_CMDS[cmd])
  395. if fname:
  396. args += ['-f', fname]
  397. if subpart:
  398. args += ['-n', subpart]
  399. if entry_name:
  400. args += ['-d', '-e', entry_name]
  401. Run(*args)
  402. def ToHex(val):
  403. """Convert an integer value (or None) to a string
  404. Returns:
  405. hex value, or 'None' if the value is None
  406. """
  407. return 'None' if val is None else '%#x' % val
  408. def ToHexSize(val):
  409. """Return the size of an object in hex
  410. Returns:
  411. hex value of size, or 'None' if the value is None
  412. """
  413. return 'None' if val is None else '%#x' % len(val)