gn_helpers.py 16 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542
  1. # Copyright 2014 The Chromium Authors. All rights reserved.
  2. # Use of this source code is governed by a BSD-style license that can be
  3. # found in the LICENSE file.
  4. """Helper functions useful when writing scripts that integrate with GN.
  5. The main functions are ToGNString() and FromGNString(), to convert between
  6. serialized GN veriables and Python variables.
  7. To use in an arbitrary Python file in the build:
  8. import os
  9. import sys
  10. sys.path.append(os.path.join(os.path.dirname(__file__),
  11. os.pardir, os.pardir, 'build'))
  12. import gn_helpers
  13. Where the sequence of parameters to join is the relative path from your source
  14. file to the build directory.
  15. """
  16. import json
  17. import os
  18. import re
  19. import sys
  20. _CHROMIUM_ROOT = os.path.join(os.path.dirname(__file__), os.pardir)
  21. BUILD_VARS_FILENAME = 'build_vars.json'
  22. IMPORT_RE = re.compile(r'^import\("//(\S+)"\)')
  23. class GNError(Exception):
  24. pass
  25. # Computes ASCII code of an element of encoded Python 2 str / Python 3 bytes.
  26. _Ord = ord if sys.version_info.major < 3 else lambda c: c
  27. def _TranslateToGnChars(s):
  28. for decoded_ch in s.encode('utf-8'): # str in Python 2, bytes in Python 3.
  29. code = _Ord(decoded_ch) # int
  30. if code in (34, 36, 92): # For '"', '$', or '\\'.
  31. yield '\\' + chr(code)
  32. elif 32 <= code < 127:
  33. yield chr(code)
  34. else:
  35. yield '$0x%02X' % code
  36. def ToGNString(value, pretty=False):
  37. """Returns a stringified GN equivalent of a Python value.
  38. Args:
  39. value: The Python value to convert.
  40. pretty: Whether to pretty print. If true, then non-empty lists are rendered
  41. recursively with one item per line, with indents. Otherwise lists are
  42. rendered without new line.
  43. Returns:
  44. The stringified GN equivalent to |value|.
  45. Raises:
  46. GNError: |value| cannot be printed to GN.
  47. """
  48. if sys.version_info.major < 3:
  49. basestring_compat = basestring
  50. else:
  51. basestring_compat = str
  52. # Emits all output tokens without intervening whitespaces.
  53. def GenerateTokens(v, level):
  54. if isinstance(v, basestring_compat):
  55. yield '"' + ''.join(_TranslateToGnChars(v)) + '"'
  56. elif isinstance(v, bool):
  57. yield 'true' if v else 'false'
  58. elif isinstance(v, int):
  59. yield str(v)
  60. elif isinstance(v, list):
  61. yield '['
  62. for i, item in enumerate(v):
  63. if i > 0:
  64. yield ','
  65. for tok in GenerateTokens(item, level + 1):
  66. yield tok
  67. yield ']'
  68. elif isinstance(v, dict):
  69. if level > 0:
  70. yield '{'
  71. for key in sorted(v):
  72. if not isinstance(key, basestring_compat):
  73. raise GNError('Dictionary key is not a string.')
  74. if not key or key[0].isdigit() or not key.replace('_', '').isalnum():
  75. raise GNError('Dictionary key is not a valid GN identifier.')
  76. yield key # No quotations.
  77. yield '='
  78. for tok in GenerateTokens(v[key], level + 1):
  79. yield tok
  80. if level > 0:
  81. yield '}'
  82. else: # Not supporting float: Add only when needed.
  83. raise GNError('Unsupported type when printing to GN.')
  84. can_start = lambda tok: tok and tok not in ',}]='
  85. can_end = lambda tok: tok and tok not in ',{[='
  86. # Adds whitespaces, trying to keep everything (except dicts) in 1 line.
  87. def PlainGlue(gen):
  88. prev_tok = None
  89. for i, tok in enumerate(gen):
  90. if i > 0:
  91. if can_end(prev_tok) and can_start(tok):
  92. yield '\n' # New dict item.
  93. elif prev_tok == '[' and tok == ']':
  94. yield ' ' # Special case for [].
  95. elif tok != ',':
  96. yield ' '
  97. yield tok
  98. prev_tok = tok
  99. # Adds whitespaces so non-empty lists can span multiple lines, with indent.
  100. def PrettyGlue(gen):
  101. prev_tok = None
  102. level = 0
  103. for i, tok in enumerate(gen):
  104. if i > 0:
  105. if can_end(prev_tok) and can_start(tok):
  106. yield '\n' + ' ' * level # New dict item.
  107. elif tok == '=' or prev_tok in '=':
  108. yield ' ' # Separator before and after '=', on same line.
  109. if tok in ']}':
  110. level -= 1
  111. # Exclude '[]' and '{}' cases.
  112. if int(prev_tok == '[') + int(tok == ']') == 1 or \
  113. int(prev_tok == '{') + int(tok == '}') == 1:
  114. yield '\n' + ' ' * level
  115. yield tok
  116. if tok in '[{':
  117. level += 1
  118. if tok == ',':
  119. yield '\n' + ' ' * level
  120. prev_tok = tok
  121. token_gen = GenerateTokens(value, 0)
  122. ret = ''.join((PrettyGlue if pretty else PlainGlue)(token_gen))
  123. # Add terminating '\n' for dict |value| or multi-line output.
  124. if isinstance(value, dict) or '\n' in ret:
  125. return ret + '\n'
  126. return ret
  127. def FromGNString(input_string):
  128. """Converts the input string from a GN serialized value to Python values.
  129. For details on supported types see GNValueParser.Parse() below.
  130. If your GN script did:
  131. something = [ "file1", "file2" ]
  132. args = [ "--values=$something" ]
  133. The command line would look something like:
  134. --values="[ \"file1\", \"file2\" ]"
  135. Which when interpreted as a command line gives the value:
  136. [ "file1", "file2" ]
  137. You can parse this into a Python list using GN rules with:
  138. input_values = FromGNValues(options.values)
  139. Although the Python 'ast' module will parse many forms of such input, it
  140. will not handle GN escaping properly, nor GN booleans. You should use this
  141. function instead.
  142. A NOTE ON STRING HANDLING:
  143. If you just pass a string on the command line to your Python script, or use
  144. string interpolation on a string variable, the strings will not be quoted:
  145. str = "asdf"
  146. args = [ str, "--value=$str" ]
  147. Will yield the command line:
  148. asdf --value=asdf
  149. The unquoted asdf string will not be valid input to this function, which
  150. accepts only quoted strings like GN scripts. In such cases, you can just use
  151. the Python string literal directly.
  152. The main use cases for this is for other types, in particular lists. When
  153. using string interpolation on a list (as in the top example) the embedded
  154. strings will be quoted and escaped according to GN rules so the list can be
  155. re-parsed to get the same result.
  156. """
  157. parser = GNValueParser(input_string)
  158. return parser.Parse()
  159. def FromGNArgs(input_string):
  160. """Converts a string with a bunch of gn arg assignments into a Python dict.
  161. Given a whitespace-separated list of
  162. <ident> = (integer | string | boolean | <list of the former>)
  163. gn assignments, this returns a Python dict, i.e.:
  164. FromGNArgs('foo=true\nbar=1\n') -> { 'foo': True, 'bar': 1 }.
  165. Only simple types and lists supported; variables, structs, calls
  166. and other, more complicated things are not.
  167. This routine is meant to handle only the simple sorts of values that
  168. arise in parsing --args.
  169. """
  170. parser = GNValueParser(input_string)
  171. return parser.ParseArgs()
  172. def UnescapeGNString(value):
  173. """Given a string with GN escaping, returns the unescaped string.
  174. Be careful not to feed with input from a Python parsing function like
  175. 'ast' because it will do Python unescaping, which will be incorrect when
  176. fed into the GN unescaper.
  177. Args:
  178. value: Input string to unescape.
  179. """
  180. result = ''
  181. i = 0
  182. while i < len(value):
  183. if value[i] == '\\':
  184. if i < len(value) - 1:
  185. next_char = value[i + 1]
  186. if next_char in ('$', '"', '\\'):
  187. # These are the escaped characters GN supports.
  188. result += next_char
  189. i += 1
  190. else:
  191. # Any other backslash is a literal.
  192. result += '\\'
  193. else:
  194. result += value[i]
  195. i += 1
  196. return result
  197. def _IsDigitOrMinus(char):
  198. return char in '-0123456789'
  199. class GNValueParser(object):
  200. """Duplicates GN parsing of values and converts to Python types.
  201. Normally you would use the wrapper function FromGNValue() below.
  202. If you expect input as a specific type, you can also call one of the Parse*
  203. functions directly. All functions throw GNError on invalid input.
  204. """
  205. def __init__(self, string, checkout_root=_CHROMIUM_ROOT):
  206. self.input = string
  207. self.cur = 0
  208. self.checkout_root = checkout_root
  209. def IsDone(self):
  210. return self.cur == len(self.input)
  211. def ReplaceImports(self):
  212. """Replaces import(...) lines with the contents of the imports.
  213. Recurses on itself until there are no imports remaining, in the case of
  214. nested imports.
  215. """
  216. lines = self.input.splitlines()
  217. if not any(line.startswith('import(') for line in lines):
  218. return
  219. for line in lines:
  220. if not line.startswith('import('):
  221. continue
  222. regex_match = IMPORT_RE.match(line)
  223. if not regex_match:
  224. raise GNError('Not a valid import string: %s' % line)
  225. import_path = os.path.join(self.checkout_root, regex_match.group(1))
  226. with open(import_path) as f:
  227. imported_args = f.read()
  228. self.input = self.input.replace(line, imported_args)
  229. # Call ourselves again if we've just replaced an import() with additional
  230. # imports.
  231. self.ReplaceImports()
  232. def _ConsumeWhitespace(self):
  233. while not self.IsDone() and self.input[self.cur] in ' \t\n':
  234. self.cur += 1
  235. def ConsumeCommentAndWhitespace(self):
  236. self._ConsumeWhitespace()
  237. # Consume each comment, line by line.
  238. while not self.IsDone() and self.input[self.cur] == '#':
  239. # Consume the rest of the comment, up until the end of the line.
  240. while not self.IsDone() and self.input[self.cur] != '\n':
  241. self.cur += 1
  242. # Move the cursor to the next line (if there is one).
  243. if not self.IsDone():
  244. self.cur += 1
  245. self._ConsumeWhitespace()
  246. def Parse(self):
  247. """Converts a string representing a printed GN value to the Python type.
  248. See additional usage notes on FromGNString() above.
  249. * GN booleans ('true', 'false') will be converted to Python booleans.
  250. * GN numbers ('123') will be converted to Python numbers.
  251. * GN strings (double-quoted as in '"asdf"') will be converted to Python
  252. strings with GN escaping rules. GN string interpolation (embedded
  253. variables preceded by $) are not supported and will be returned as
  254. literals.
  255. * GN lists ('[1, "asdf", 3]') will be converted to Python lists.
  256. * GN scopes ('{ ... }') are not supported.
  257. Raises:
  258. GNError: Parse fails.
  259. """
  260. result = self._ParseAllowTrailing()
  261. self.ConsumeCommentAndWhitespace()
  262. if not self.IsDone():
  263. raise GNError("Trailing input after parsing:\n " + self.input[self.cur:])
  264. return result
  265. def ParseArgs(self):
  266. """Converts a whitespace-separated list of ident=literals to a dict.
  267. See additional usage notes on FromGNArgs(), above.
  268. Raises:
  269. GNError: Parse fails.
  270. """
  271. d = {}
  272. self.ReplaceImports()
  273. self.ConsumeCommentAndWhitespace()
  274. while not self.IsDone():
  275. ident = self._ParseIdent()
  276. self.ConsumeCommentAndWhitespace()
  277. if self.input[self.cur] != '=':
  278. raise GNError("Unexpected token: " + self.input[self.cur:])
  279. self.cur += 1
  280. self.ConsumeCommentAndWhitespace()
  281. val = self._ParseAllowTrailing()
  282. self.ConsumeCommentAndWhitespace()
  283. d[ident] = val
  284. return d
  285. def _ParseAllowTrailing(self):
  286. """Internal version of Parse() that doesn't check for trailing stuff."""
  287. self.ConsumeCommentAndWhitespace()
  288. if self.IsDone():
  289. raise GNError("Expected input to parse.")
  290. next_char = self.input[self.cur]
  291. if next_char == '[':
  292. return self.ParseList()
  293. elif next_char == '{':
  294. return self.ParseScope()
  295. elif _IsDigitOrMinus(next_char):
  296. return self.ParseNumber()
  297. elif next_char == '"':
  298. return self.ParseString()
  299. elif self._ConstantFollows('true'):
  300. return True
  301. elif self._ConstantFollows('false'):
  302. return False
  303. else:
  304. raise GNError("Unexpected token: " + self.input[self.cur:])
  305. def _ParseIdent(self):
  306. ident = ''
  307. next_char = self.input[self.cur]
  308. if not next_char.isalpha() and not next_char=='_':
  309. raise GNError("Expected an identifier: " + self.input[self.cur:])
  310. ident += next_char
  311. self.cur += 1
  312. next_char = self.input[self.cur]
  313. while next_char.isalpha() or next_char.isdigit() or next_char=='_':
  314. ident += next_char
  315. self.cur += 1
  316. next_char = self.input[self.cur]
  317. return ident
  318. def ParseNumber(self):
  319. self.ConsumeCommentAndWhitespace()
  320. if self.IsDone():
  321. raise GNError('Expected number but got nothing.')
  322. begin = self.cur
  323. # The first character can include a negative sign.
  324. if not self.IsDone() and _IsDigitOrMinus(self.input[self.cur]):
  325. self.cur += 1
  326. while not self.IsDone() and self.input[self.cur].isdigit():
  327. self.cur += 1
  328. number_string = self.input[begin:self.cur]
  329. if not len(number_string) or number_string == '-':
  330. raise GNError('Not a valid number.')
  331. return int(number_string)
  332. def ParseString(self):
  333. self.ConsumeCommentAndWhitespace()
  334. if self.IsDone():
  335. raise GNError('Expected string but got nothing.')
  336. if self.input[self.cur] != '"':
  337. raise GNError('Expected string beginning in a " but got:\n ' +
  338. self.input[self.cur:])
  339. self.cur += 1 # Skip over quote.
  340. begin = self.cur
  341. while not self.IsDone() and self.input[self.cur] != '"':
  342. if self.input[self.cur] == '\\':
  343. self.cur += 1 # Skip over the backslash.
  344. if self.IsDone():
  345. raise GNError('String ends in a backslash in:\n ' + self.input)
  346. self.cur += 1
  347. if self.IsDone():
  348. raise GNError('Unterminated string:\n ' + self.input[begin:])
  349. end = self.cur
  350. self.cur += 1 # Consume trailing ".
  351. return UnescapeGNString(self.input[begin:end])
  352. def ParseList(self):
  353. self.ConsumeCommentAndWhitespace()
  354. if self.IsDone():
  355. raise GNError('Expected list but got nothing.')
  356. # Skip over opening '['.
  357. if self.input[self.cur] != '[':
  358. raise GNError('Expected [ for list but got:\n ' + self.input[self.cur:])
  359. self.cur += 1
  360. self.ConsumeCommentAndWhitespace()
  361. if self.IsDone():
  362. raise GNError('Unterminated list:\n ' + self.input)
  363. list_result = []
  364. previous_had_trailing_comma = True
  365. while not self.IsDone():
  366. if self.input[self.cur] == ']':
  367. self.cur += 1 # Skip over ']'.
  368. return list_result
  369. if not previous_had_trailing_comma:
  370. raise GNError('List items not separated by comma.')
  371. list_result += [ self._ParseAllowTrailing() ]
  372. self.ConsumeCommentAndWhitespace()
  373. if self.IsDone():
  374. break
  375. # Consume comma if there is one.
  376. previous_had_trailing_comma = self.input[self.cur] == ','
  377. if previous_had_trailing_comma:
  378. # Consume comma.
  379. self.cur += 1
  380. self.ConsumeCommentAndWhitespace()
  381. raise GNError('Unterminated list:\n ' + self.input)
  382. def ParseScope(self):
  383. self.ConsumeCommentAndWhitespace()
  384. if self.IsDone():
  385. raise GNError('Expected scope but got nothing.')
  386. # Skip over opening '{'.
  387. if self.input[self.cur] != '{':
  388. raise GNError('Expected { for scope but got:\n ' + self.input[self.cur:])
  389. self.cur += 1
  390. self.ConsumeCommentAndWhitespace()
  391. if self.IsDone():
  392. raise GNError('Unterminated scope:\n ' + self.input)
  393. scope_result = {}
  394. while not self.IsDone():
  395. if self.input[self.cur] == '}':
  396. self.cur += 1
  397. return scope_result
  398. ident = self._ParseIdent()
  399. self.ConsumeCommentAndWhitespace()
  400. if self.input[self.cur] != '=':
  401. raise GNError("Unexpected token: " + self.input[self.cur:])
  402. self.cur += 1
  403. self.ConsumeCommentAndWhitespace()
  404. val = self._ParseAllowTrailing()
  405. self.ConsumeCommentAndWhitespace()
  406. scope_result[ident] = val
  407. raise GNError('Unterminated scope:\n ' + self.input)
  408. def _ConstantFollows(self, constant):
  409. """Checks and maybe consumes a string constant at current input location.
  410. Param:
  411. constant: The string constant to check.
  412. Returns:
  413. True if |constant| follows immediately at the current location in the
  414. input. In this case, the string is consumed as a side effect. Otherwise,
  415. returns False and the current position is unchanged.
  416. """
  417. end = self.cur + len(constant)
  418. if end > len(self.input):
  419. return False # Not enough room.
  420. if self.input[self.cur:end] == constant:
  421. self.cur = end
  422. return True
  423. return False
  424. def ReadBuildVars(output_directory):
  425. """Parses $output_directory/build_vars.json into a dict."""
  426. with open(os.path.join(output_directory, BUILD_VARS_FILENAME)) as f:
  427. return json.load(f)