pak_util.py 9.7 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270
  1. #!/usr/bin/env python3
  2. # Copyright 2017 The Chromium Authors. All rights reserved.
  3. # Use of this source code is governed by a BSD-style license that can be
  4. # found in the LICENSE file.
  5. """A tool for interacting with .pak files.
  6. For details on the pak file format, see:
  7. https://dev.chromium.org/developers/design-documents/linuxresourcesandlocalizedstrings
  8. """
  9. from __future__ import print_function
  10. import argparse
  11. import gzip
  12. import hashlib
  13. import os
  14. import shutil
  15. import subprocess
  16. import sys
  17. import tempfile
  18. _HERE_PATH = os.path.dirname(__file__)
  19. _SRC_PATH = os.path.normpath(os.path.join(_HERE_PATH, '..', '..'))
  20. sys.path.insert(0, os.path.join(_SRC_PATH, 'third_party', 'six', 'src'))
  21. import six
  22. from grit import constants
  23. from grit.format import data_pack
  24. _GZIP_HEADER = b'\x1f\x8b'
  25. def _RepackMain(args):
  26. output_info_filepath = args.output_pak_file + '.info'
  27. if args.compress:
  28. # If the file needs to be compressed, call RePack with a tempfile path,
  29. # then compress the tempfile to args.output_pak_file.
  30. temp_outfile = tempfile.NamedTemporaryFile()
  31. out_path = temp_outfile.name
  32. # Strip any non .pak extension from the .info output file path.
  33. splitext = os.path.splitext(args.output_pak_file)
  34. if splitext[1] != '.pak':
  35. output_info_filepath = splitext[0] + '.info'
  36. else:
  37. out_path = args.output_pak_file
  38. data_pack.RePack(out_path,
  39. args.input_pak_files,
  40. args.allowlist,
  41. args.suppress_removed_key_output,
  42. output_info_filepath=output_info_filepath)
  43. if args.compress:
  44. with open(args.output_pak_file, 'wb') as out:
  45. with gzip.GzipFile(filename='', mode='wb', fileobj=out, mtime=0) as outgz:
  46. shutil.copyfileobj(temp_outfile, outgz)
  47. def _MaybeDecompress(payload, brotli_path=None):
  48. if payload.startswith(_GZIP_HEADER):
  49. return gzip.decompress(payload)
  50. if payload.startswith(constants.BROTLI_CONST):
  51. shell = brotli_path is None
  52. brotli_path = brotli_path or 'brotli'
  53. # Header is 2 bytes, size is 6 bytes.
  54. payload = payload[8:]
  55. try:
  56. result = subprocess.run([brotli_path, '--decompress', '--stdout'],
  57. shell=shell,
  58. input=payload,
  59. stdout=subprocess.PIPE,
  60. check=True)
  61. # I don't know why with "sudo apt-get install brotli", files come out 4
  62. # bytes larger and the command doesn't fail.
  63. if len(result.stdout) == len(payload) + 4:
  64. sys.stderr.write('Brotli decompression failed. You likely need to use '
  65. 'the version of brotli built by Chrome '
  66. '(out/Release/clang_x64/brotli).\n')
  67. sys.exit(1)
  68. return result.stdout
  69. except subprocess.CalledProcessError as e:
  70. sys.stderr.write(str(e) + '\n')
  71. sys.exit(1)
  72. return payload
  73. def _ExtractMain(args):
  74. pak = data_pack.ReadDataPack(args.pak_file)
  75. if args.textual_id:
  76. info_dict = data_pack.ReadGrdInfo(args.pak_file)
  77. for resource_id, payload in pak.resources.items():
  78. filename = (
  79. info_dict[resource_id].textual_id
  80. if args.textual_id else str(resource_id))
  81. path = os.path.join(args.output_dir, filename)
  82. with open(path, 'wb') as f:
  83. if not args.raw:
  84. payload = _MaybeDecompress(payload, args.brotli)
  85. f.write(payload)
  86. def _CreateMain(args):
  87. pak = {}
  88. for name in os.listdir(args.input_dir):
  89. try:
  90. resource_id = int(name)
  91. except:
  92. continue
  93. filename = os.path.join(args.input_dir, name)
  94. if os.path.isfile(filename):
  95. with open(filename, 'rb') as f:
  96. pak[resource_id] = f.read()
  97. data_pack.WriteDataPack(pak, args.output_pak_file, data_pack.UTF8)
  98. def _PrintMain(args):
  99. pak = data_pack.ReadDataPack(args.pak_file)
  100. if args.textual_id:
  101. info_dict = data_pack.ReadGrdInfo(args.pak_file)
  102. output = args.output
  103. encoding = 'binary'
  104. if pak.encoding == 1:
  105. encoding = 'utf-8'
  106. elif pak.encoding == 2:
  107. encoding = 'utf-16'
  108. else:
  109. encoding = '?' + str(pak.encoding)
  110. output.write('version: {}\n'.format(pak.version))
  111. output.write('encoding: {}\n'.format(encoding))
  112. output.write('num_resources: {}\n'.format(len(pak.resources)))
  113. output.write('num_aliases: {}\n'.format(len(pak.aliases)))
  114. breakdown = ', '.join('{}: {}'.format(*x) for x in pak.sizes)
  115. output.write('total_size: {} ({})\n'.format(pak.sizes.total, breakdown))
  116. try_decode = args.decode and encoding.startswith('utf')
  117. # Print IDs in ascending order, since that's the order in which they appear in
  118. # the file (order is lost by Python dict).
  119. for resource_id in sorted(pak.resources):
  120. data = pak.resources[resource_id]
  121. canonical_id = pak.aliases.get(resource_id, resource_id)
  122. desc = '<data>'
  123. if try_decode:
  124. try:
  125. desc = six.text_type(data, encoding)
  126. if len(desc) > 60:
  127. desc = desc[:60] + '...'
  128. desc = desc.replace('\n', '\\n')
  129. except UnicodeDecodeError:
  130. pass
  131. sha1 = hashlib.sha1(data).hexdigest()[:10]
  132. if args.textual_id:
  133. textual_id = info_dict[resource_id].textual_id
  134. canonical_textual_id = info_dict[canonical_id].textual_id
  135. output.write(
  136. 'Entry(id={}, canonical_id={}, size={}, sha1={}): {}\n'.format(
  137. textual_id, canonical_textual_id, len(data), sha1,
  138. desc))
  139. else:
  140. output.write(
  141. 'Entry(id={}, canonical_id={}, size={}, sha1={}): {}\n'.format(
  142. resource_id, canonical_id, len(data), sha1, desc))
  143. def _ListMain(args):
  144. pak = data_pack.ReadDataPack(args.pak_file)
  145. if args.textual_id or args.path:
  146. info_dict = data_pack.ReadGrdInfo(args.pak_file)
  147. fmt = ''.join([
  148. '{id}', ' = {textual_id}' if args.textual_id else '',
  149. ' @ {path}' if args.path else '', '\n'
  150. ])
  151. for resource_id in sorted(pak.resources):
  152. item = info_dict[resource_id]
  153. args.output.write(
  154. fmt.format(textual_id=item.textual_id, id=item.id, path=item.path))
  155. else:
  156. for resource_id in sorted(pak.resources):
  157. args.output.write('%d\n' % resource_id)
  158. def main():
  159. parser = argparse.ArgumentParser(
  160. description=__doc__, formatter_class=argparse.RawTextHelpFormatter)
  161. # Subparsers are required by default under Python 2. Python 3 changed to
  162. # not required, but didn't include a required option until 3.7. Setting
  163. # the required member works in all versions (and setting dest name).
  164. sub_parsers = parser.add_subparsers(dest='action')
  165. sub_parsers.required = True
  166. sub_parser = sub_parsers.add_parser('repack',
  167. help='Combines several .pak files into one.')
  168. sub_parser.add_argument('output_pak_file', help='File to create.')
  169. sub_parser.add_argument('input_pak_files', nargs='+',
  170. help='Input .pak files.')
  171. sub_parser.add_argument(
  172. '--allowlist',
  173. help='Path to a allowlist used to filter output pak file resource IDs.')
  174. sub_parser.add_argument(
  175. '--suppress-removed-key-output',
  176. action='store_true',
  177. help='Do not log which keys were removed by the allowlist.')
  178. sub_parser.add_argument('--compress', dest='compress', action='store_true',
  179. default=False, help='Compress output_pak_file using gzip.')
  180. sub_parser.set_defaults(func=_RepackMain)
  181. sub_parser = sub_parsers.add_parser('extract', help='Extracts pak file')
  182. sub_parser.add_argument('pak_file')
  183. sub_parser.add_argument('-o', '--output-dir', default='.',
  184. help='Directory to extract to.')
  185. sub_parser.add_argument('--raw',
  186. action='store_true',
  187. help='Do not decompress when extracting.')
  188. sub_parser.add_argument('--brotli',
  189. help='Path to brotli executable. Needed only to '
  190. 'decompress brotli-compressed entries. For a '
  191. 'chromium checkout, find in your output directory')
  192. sub_parser.add_argument(
  193. '-t',
  194. '--textual-id',
  195. action='store_true',
  196. help='Use textual resource ID (name) (from .info file) as filenames.')
  197. sub_parser.set_defaults(func=_ExtractMain)
  198. sub_parser = sub_parsers.add_parser('create',
  199. help='Creates pak file from extracted directory.')
  200. sub_parser.add_argument('output_pak_file', help='File to create.')
  201. sub_parser.add_argument('-i', '--input-dir', default='.',
  202. help='Directory to create from.')
  203. sub_parser.set_defaults(func=_CreateMain)
  204. sub_parser = sub_parsers.add_parser('print',
  205. help='Prints all pak IDs and contents. Useful for diffing.')
  206. sub_parser.add_argument('pak_file')
  207. sub_parser.add_argument('--output', type=argparse.FileType('w'),
  208. default=sys.stdout,
  209. help='The resource list path to write (default stdout)')
  210. sub_parser.add_argument('--no-decode', dest='decode', action='store_false',
  211. default=True, help='Do not print entry data.')
  212. sub_parser.add_argument(
  213. '-t',
  214. '--textual-id',
  215. action='store_true',
  216. help='Print textual ID (name) (from .info file) instead of the ID.')
  217. sub_parser.set_defaults(func=_PrintMain)
  218. sub_parser = sub_parsers.add_parser('list-id',
  219. help='Outputs all resource IDs to a file.')
  220. sub_parser.add_argument('pak_file')
  221. sub_parser.add_argument('--output', type=argparse.FileType('w'),
  222. default=sys.stdout,
  223. help='The resource list path to write (default stdout)')
  224. sub_parser.add_argument(
  225. '-t',
  226. '--textual-id',
  227. action='store_true',
  228. help='Print the textual resource ID (from .info file).')
  229. sub_parser.add_argument(
  230. '-p',
  231. '--path',
  232. action='store_true',
  233. help='Print the resource path (from .info file).')
  234. sub_parser.set_defaults(func=_ListMain)
  235. args = parser.parse_args()
  236. args.func(args)
  237. if __name__ == '__main__':
  238. main()