123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542 |
- # Copyright 2014 The Chromium Authors. All rights reserved.
- # Use of this source code is governed by a BSD-style license that can be
- # found in the LICENSE file.
- """Helper functions useful when writing scripts that integrate with GN.
- The main functions are ToGNString() and FromGNString(), to convert between
- serialized GN veriables and Python variables.
- To use in an arbitrary Python file in the build:
- import os
- import sys
- sys.path.append(os.path.join(os.path.dirname(__file__),
- os.pardir, os.pardir, 'build'))
- import gn_helpers
- Where the sequence of parameters to join is the relative path from your source
- file to the build directory.
- """
- import json
- import os
- import re
- import sys
- _CHROMIUM_ROOT = os.path.join(os.path.dirname(__file__), os.pardir)
- BUILD_VARS_FILENAME = 'build_vars.json'
- IMPORT_RE = re.compile(r'^import\("//(\S+)"\)')
- class GNError(Exception):
- pass
- # Computes ASCII code of an element of encoded Python 2 str / Python 3 bytes.
- _Ord = ord if sys.version_info.major < 3 else lambda c: c
- def _TranslateToGnChars(s):
- for decoded_ch in s.encode('utf-8'): # str in Python 2, bytes in Python 3.
- code = _Ord(decoded_ch) # int
- if code in (34, 36, 92): # For '"', '$', or '\\'.
- yield '\\' + chr(code)
- elif 32 <= code < 127:
- yield chr(code)
- else:
- yield '$0x%02X' % code
- def ToGNString(value, pretty=False):
- """Returns a stringified GN equivalent of a Python value.
- Args:
- value: The Python value to convert.
- pretty: Whether to pretty print. If true, then non-empty lists are rendered
- recursively with one item per line, with indents. Otherwise lists are
- rendered without new line.
- Returns:
- The stringified GN equivalent to |value|.
- Raises:
- GNError: |value| cannot be printed to GN.
- """
- if sys.version_info.major < 3:
- basestring_compat = basestring
- else:
- basestring_compat = str
- # Emits all output tokens without intervening whitespaces.
- def GenerateTokens(v, level):
- if isinstance(v, basestring_compat):
- yield '"' + ''.join(_TranslateToGnChars(v)) + '"'
- elif isinstance(v, bool):
- yield 'true' if v else 'false'
- elif isinstance(v, int):
- yield str(v)
- elif isinstance(v, list):
- yield '['
- for i, item in enumerate(v):
- if i > 0:
- yield ','
- for tok in GenerateTokens(item, level + 1):
- yield tok
- yield ']'
- elif isinstance(v, dict):
- if level > 0:
- yield '{'
- for key in sorted(v):
- if not isinstance(key, basestring_compat):
- raise GNError('Dictionary key is not a string.')
- if not key or key[0].isdigit() or not key.replace('_', '').isalnum():
- raise GNError('Dictionary key is not a valid GN identifier.')
- yield key # No quotations.
- yield '='
- for tok in GenerateTokens(v[key], level + 1):
- yield tok
- if level > 0:
- yield '}'
- else: # Not supporting float: Add only when needed.
- raise GNError('Unsupported type when printing to GN.')
- can_start = lambda tok: tok and tok not in ',}]='
- can_end = lambda tok: tok and tok not in ',{[='
- # Adds whitespaces, trying to keep everything (except dicts) in 1 line.
- def PlainGlue(gen):
- prev_tok = None
- for i, tok in enumerate(gen):
- if i > 0:
- if can_end(prev_tok) and can_start(tok):
- yield '\n' # New dict item.
- elif prev_tok == '[' and tok == ']':
- yield ' ' # Special case for [].
- elif tok != ',':
- yield ' '
- yield tok
- prev_tok = tok
- # Adds whitespaces so non-empty lists can span multiple lines, with indent.
- def PrettyGlue(gen):
- prev_tok = None
- level = 0
- for i, tok in enumerate(gen):
- if i > 0:
- if can_end(prev_tok) and can_start(tok):
- yield '\n' + ' ' * level # New dict item.
- elif tok == '=' or prev_tok in '=':
- yield ' ' # Separator before and after '=', on same line.
- if tok in ']}':
- level -= 1
- # Exclude '[]' and '{}' cases.
- if int(prev_tok == '[') + int(tok == ']') == 1 or \
- int(prev_tok == '{') + int(tok == '}') == 1:
- yield '\n' + ' ' * level
- yield tok
- if tok in '[{':
- level += 1
- if tok == ',':
- yield '\n' + ' ' * level
- prev_tok = tok
- token_gen = GenerateTokens(value, 0)
- ret = ''.join((PrettyGlue if pretty else PlainGlue)(token_gen))
- # Add terminating '\n' for dict |value| or multi-line output.
- if isinstance(value, dict) or '\n' in ret:
- return ret + '\n'
- return ret
- def FromGNString(input_string):
- """Converts the input string from a GN serialized value to Python values.
- For details on supported types see GNValueParser.Parse() below.
- If your GN script did:
- something = [ "file1", "file2" ]
- args = [ "--values=$something" ]
- The command line would look something like:
- --values="[ \"file1\", \"file2\" ]"
- Which when interpreted as a command line gives the value:
- [ "file1", "file2" ]
- You can parse this into a Python list using GN rules with:
- input_values = FromGNValues(options.values)
- Although the Python 'ast' module will parse many forms of such input, it
- will not handle GN escaping properly, nor GN booleans. You should use this
- function instead.
- A NOTE ON STRING HANDLING:
- If you just pass a string on the command line to your Python script, or use
- string interpolation on a string variable, the strings will not be quoted:
- str = "asdf"
- args = [ str, "--value=$str" ]
- Will yield the command line:
- asdf --value=asdf
- The unquoted asdf string will not be valid input to this function, which
- accepts only quoted strings like GN scripts. In such cases, you can just use
- the Python string literal directly.
- The main use cases for this is for other types, in particular lists. When
- using string interpolation on a list (as in the top example) the embedded
- strings will be quoted and escaped according to GN rules so the list can be
- re-parsed to get the same result.
- """
- parser = GNValueParser(input_string)
- return parser.Parse()
- def FromGNArgs(input_string):
- """Converts a string with a bunch of gn arg assignments into a Python dict.
- Given a whitespace-separated list of
- <ident> = (integer | string | boolean | <list of the former>)
- gn assignments, this returns a Python dict, i.e.:
- FromGNArgs('foo=true\nbar=1\n') -> { 'foo': True, 'bar': 1 }.
- Only simple types and lists supported; variables, structs, calls
- and other, more complicated things are not.
- This routine is meant to handle only the simple sorts of values that
- arise in parsing --args.
- """
- parser = GNValueParser(input_string)
- return parser.ParseArgs()
- def UnescapeGNString(value):
- """Given a string with GN escaping, returns the unescaped string.
- Be careful not to feed with input from a Python parsing function like
- 'ast' because it will do Python unescaping, which will be incorrect when
- fed into the GN unescaper.
- Args:
- value: Input string to unescape.
- """
- result = ''
- i = 0
- while i < len(value):
- if value[i] == '\\':
- if i < len(value) - 1:
- next_char = value[i + 1]
- if next_char in ('$', '"', '\\'):
- # These are the escaped characters GN supports.
- result += next_char
- i += 1
- else:
- # Any other backslash is a literal.
- result += '\\'
- else:
- result += value[i]
- i += 1
- return result
- def _IsDigitOrMinus(char):
- return char in '-0123456789'
- class GNValueParser(object):
- """Duplicates GN parsing of values and converts to Python types.
- Normally you would use the wrapper function FromGNValue() below.
- If you expect input as a specific type, you can also call one of the Parse*
- functions directly. All functions throw GNError on invalid input.
- """
- def __init__(self, string, checkout_root=_CHROMIUM_ROOT):
- self.input = string
- self.cur = 0
- self.checkout_root = checkout_root
- def IsDone(self):
- return self.cur == len(self.input)
- def ReplaceImports(self):
- """Replaces import(...) lines with the contents of the imports.
- Recurses on itself until there are no imports remaining, in the case of
- nested imports.
- """
- lines = self.input.splitlines()
- if not any(line.startswith('import(') for line in lines):
- return
- for line in lines:
- if not line.startswith('import('):
- continue
- regex_match = IMPORT_RE.match(line)
- if not regex_match:
- raise GNError('Not a valid import string: %s' % line)
- import_path = os.path.join(self.checkout_root, regex_match.group(1))
- with open(import_path) as f:
- imported_args = f.read()
- self.input = self.input.replace(line, imported_args)
- # Call ourselves again if we've just replaced an import() with additional
- # imports.
- self.ReplaceImports()
- def _ConsumeWhitespace(self):
- while not self.IsDone() and self.input[self.cur] in ' \t\n':
- self.cur += 1
- def ConsumeCommentAndWhitespace(self):
- self._ConsumeWhitespace()
- # Consume each comment, line by line.
- while not self.IsDone() and self.input[self.cur] == '#':
- # Consume the rest of the comment, up until the end of the line.
- while not self.IsDone() and self.input[self.cur] != '\n':
- self.cur += 1
- # Move the cursor to the next line (if there is one).
- if not self.IsDone():
- self.cur += 1
- self._ConsumeWhitespace()
- def Parse(self):
- """Converts a string representing a printed GN value to the Python type.
- See additional usage notes on FromGNString() above.
- * GN booleans ('true', 'false') will be converted to Python booleans.
- * GN numbers ('123') will be converted to Python numbers.
- * GN strings (double-quoted as in '"asdf"') will be converted to Python
- strings with GN escaping rules. GN string interpolation (embedded
- variables preceded by $) are not supported and will be returned as
- literals.
- * GN lists ('[1, "asdf", 3]') will be converted to Python lists.
- * GN scopes ('{ ... }') are not supported.
- Raises:
- GNError: Parse fails.
- """
- result = self._ParseAllowTrailing()
- self.ConsumeCommentAndWhitespace()
- if not self.IsDone():
- raise GNError("Trailing input after parsing:\n " + self.input[self.cur:])
- return result
- def ParseArgs(self):
- """Converts a whitespace-separated list of ident=literals to a dict.
- See additional usage notes on FromGNArgs(), above.
- Raises:
- GNError: Parse fails.
- """
- d = {}
- self.ReplaceImports()
- self.ConsumeCommentAndWhitespace()
- while not self.IsDone():
- ident = self._ParseIdent()
- self.ConsumeCommentAndWhitespace()
- if self.input[self.cur] != '=':
- raise GNError("Unexpected token: " + self.input[self.cur:])
- self.cur += 1
- self.ConsumeCommentAndWhitespace()
- val = self._ParseAllowTrailing()
- self.ConsumeCommentAndWhitespace()
- d[ident] = val
- return d
- def _ParseAllowTrailing(self):
- """Internal version of Parse() that doesn't check for trailing stuff."""
- self.ConsumeCommentAndWhitespace()
- if self.IsDone():
- raise GNError("Expected input to parse.")
- next_char = self.input[self.cur]
- if next_char == '[':
- return self.ParseList()
- elif next_char == '{':
- return self.ParseScope()
- elif _IsDigitOrMinus(next_char):
- return self.ParseNumber()
- elif next_char == '"':
- return self.ParseString()
- elif self._ConstantFollows('true'):
- return True
- elif self._ConstantFollows('false'):
- return False
- else:
- raise GNError("Unexpected token: " + self.input[self.cur:])
- def _ParseIdent(self):
- ident = ''
- next_char = self.input[self.cur]
- if not next_char.isalpha() and not next_char=='_':
- raise GNError("Expected an identifier: " + self.input[self.cur:])
- ident += next_char
- self.cur += 1
- next_char = self.input[self.cur]
- while next_char.isalpha() or next_char.isdigit() or next_char=='_':
- ident += next_char
- self.cur += 1
- next_char = self.input[self.cur]
- return ident
- def ParseNumber(self):
- self.ConsumeCommentAndWhitespace()
- if self.IsDone():
- raise GNError('Expected number but got nothing.')
- begin = self.cur
- # The first character can include a negative sign.
- if not self.IsDone() and _IsDigitOrMinus(self.input[self.cur]):
- self.cur += 1
- while not self.IsDone() and self.input[self.cur].isdigit():
- self.cur += 1
- number_string = self.input[begin:self.cur]
- if not len(number_string) or number_string == '-':
- raise GNError('Not a valid number.')
- return int(number_string)
- def ParseString(self):
- self.ConsumeCommentAndWhitespace()
- if self.IsDone():
- raise GNError('Expected string but got nothing.')
- if self.input[self.cur] != '"':
- raise GNError('Expected string beginning in a " but got:\n ' +
- self.input[self.cur:])
- self.cur += 1 # Skip over quote.
- begin = self.cur
- while not self.IsDone() and self.input[self.cur] != '"':
- if self.input[self.cur] == '\\':
- self.cur += 1 # Skip over the backslash.
- if self.IsDone():
- raise GNError('String ends in a backslash in:\n ' + self.input)
- self.cur += 1
- if self.IsDone():
- raise GNError('Unterminated string:\n ' + self.input[begin:])
- end = self.cur
- self.cur += 1 # Consume trailing ".
- return UnescapeGNString(self.input[begin:end])
- def ParseList(self):
- self.ConsumeCommentAndWhitespace()
- if self.IsDone():
- raise GNError('Expected list but got nothing.')
- # Skip over opening '['.
- if self.input[self.cur] != '[':
- raise GNError('Expected [ for list but got:\n ' + self.input[self.cur:])
- self.cur += 1
- self.ConsumeCommentAndWhitespace()
- if self.IsDone():
- raise GNError('Unterminated list:\n ' + self.input)
- list_result = []
- previous_had_trailing_comma = True
- while not self.IsDone():
- if self.input[self.cur] == ']':
- self.cur += 1 # Skip over ']'.
- return list_result
- if not previous_had_trailing_comma:
- raise GNError('List items not separated by comma.')
- list_result += [ self._ParseAllowTrailing() ]
- self.ConsumeCommentAndWhitespace()
- if self.IsDone():
- break
- # Consume comma if there is one.
- previous_had_trailing_comma = self.input[self.cur] == ','
- if previous_had_trailing_comma:
- # Consume comma.
- self.cur += 1
- self.ConsumeCommentAndWhitespace()
- raise GNError('Unterminated list:\n ' + self.input)
- def ParseScope(self):
- self.ConsumeCommentAndWhitespace()
- if self.IsDone():
- raise GNError('Expected scope but got nothing.')
- # Skip over opening '{'.
- if self.input[self.cur] != '{':
- raise GNError('Expected { for scope but got:\n ' + self.input[self.cur:])
- self.cur += 1
- self.ConsumeCommentAndWhitespace()
- if self.IsDone():
- raise GNError('Unterminated scope:\n ' + self.input)
- scope_result = {}
- while not self.IsDone():
- if self.input[self.cur] == '}':
- self.cur += 1
- return scope_result
- ident = self._ParseIdent()
- self.ConsumeCommentAndWhitespace()
- if self.input[self.cur] != '=':
- raise GNError("Unexpected token: " + self.input[self.cur:])
- self.cur += 1
- self.ConsumeCommentAndWhitespace()
- val = self._ParseAllowTrailing()
- self.ConsumeCommentAndWhitespace()
- scope_result[ident] = val
- raise GNError('Unterminated scope:\n ' + self.input)
- def _ConstantFollows(self, constant):
- """Checks and maybe consumes a string constant at current input location.
- Param:
- constant: The string constant to check.
- Returns:
- True if |constant| follows immediately at the current location in the
- input. In this case, the string is consumed as a side effect. Otherwise,
- returns False and the current position is unchanged.
- """
- end = self.cur + len(constant)
- if end > len(self.input):
- return False # Not enough room.
- if self.input[self.cur:end] == constant:
- self.cur = end
- return True
- return False
- def ReadBuildVars(output_directory):
- """Parses $output_directory/build_vars.json into a dict."""
- with open(os.path.join(output_directory, BUILD_VARS_FILENAME)) as f:
- return json.load(f)
|