ext.py 26 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704
  1. # -*- coding: utf-8 -*-
  2. """Extension API for adding custom tags and behavior."""
  3. import pprint
  4. import re
  5. from sys import version_info
  6. from markupsafe import Markup
  7. from . import nodes
  8. from ._compat import iteritems
  9. from ._compat import string_types
  10. from ._compat import with_metaclass
  11. from .defaults import BLOCK_END_STRING
  12. from .defaults import BLOCK_START_STRING
  13. from .defaults import COMMENT_END_STRING
  14. from .defaults import COMMENT_START_STRING
  15. from .defaults import KEEP_TRAILING_NEWLINE
  16. from .defaults import LINE_COMMENT_PREFIX
  17. from .defaults import LINE_STATEMENT_PREFIX
  18. from .defaults import LSTRIP_BLOCKS
  19. from .defaults import NEWLINE_SEQUENCE
  20. from .defaults import TRIM_BLOCKS
  21. from .defaults import VARIABLE_END_STRING
  22. from .defaults import VARIABLE_START_STRING
  23. from .environment import Environment
  24. from .exceptions import TemplateAssertionError
  25. from .exceptions import TemplateSyntaxError
  26. from .nodes import ContextReference
  27. from .runtime import concat
  28. from .utils import contextfunction
  29. from .utils import import_string
  30. # the only real useful gettext functions for a Jinja template. Note
  31. # that ugettext must be assigned to gettext as Jinja doesn't support
  32. # non unicode strings.
  33. GETTEXT_FUNCTIONS = ("_", "gettext", "ngettext")
  34. _ws_re = re.compile(r"\s*\n\s*")
  35. class ExtensionRegistry(type):
  36. """Gives the extension an unique identifier."""
  37. def __new__(mcs, name, bases, d):
  38. rv = type.__new__(mcs, name, bases, d)
  39. rv.identifier = rv.__module__ + "." + rv.__name__
  40. return rv
  41. class Extension(with_metaclass(ExtensionRegistry, object)):
  42. """Extensions can be used to add extra functionality to the Jinja template
  43. system at the parser level. Custom extensions are bound to an environment
  44. but may not store environment specific data on `self`. The reason for
  45. this is that an extension can be bound to another environment (for
  46. overlays) by creating a copy and reassigning the `environment` attribute.
  47. As extensions are created by the environment they cannot accept any
  48. arguments for configuration. One may want to work around that by using
  49. a factory function, but that is not possible as extensions are identified
  50. by their import name. The correct way to configure the extension is
  51. storing the configuration values on the environment. Because this way the
  52. environment ends up acting as central configuration storage the
  53. attributes may clash which is why extensions have to ensure that the names
  54. they choose for configuration are not too generic. ``prefix`` for example
  55. is a terrible name, ``fragment_cache_prefix`` on the other hand is a good
  56. name as includes the name of the extension (fragment cache).
  57. """
  58. #: if this extension parses this is the list of tags it's listening to.
  59. tags = set()
  60. #: the priority of that extension. This is especially useful for
  61. #: extensions that preprocess values. A lower value means higher
  62. #: priority.
  63. #:
  64. #: .. versionadded:: 2.4
  65. priority = 100
  66. def __init__(self, environment):
  67. self.environment = environment
  68. def bind(self, environment):
  69. """Create a copy of this extension bound to another environment."""
  70. rv = object.__new__(self.__class__)
  71. rv.__dict__.update(self.__dict__)
  72. rv.environment = environment
  73. return rv
  74. def preprocess(self, source, name, filename=None):
  75. """This method is called before the actual lexing and can be used to
  76. preprocess the source. The `filename` is optional. The return value
  77. must be the preprocessed source.
  78. """
  79. return source
  80. def filter_stream(self, stream):
  81. """It's passed a :class:`~jinja2.lexer.TokenStream` that can be used
  82. to filter tokens returned. This method has to return an iterable of
  83. :class:`~jinja2.lexer.Token`\\s, but it doesn't have to return a
  84. :class:`~jinja2.lexer.TokenStream`.
  85. """
  86. return stream
  87. def parse(self, parser):
  88. """If any of the :attr:`tags` matched this method is called with the
  89. parser as first argument. The token the parser stream is pointing at
  90. is the name token that matched. This method has to return one or a
  91. list of multiple nodes.
  92. """
  93. raise NotImplementedError()
  94. def attr(self, name, lineno=None):
  95. """Return an attribute node for the current extension. This is useful
  96. to pass constants on extensions to generated template code.
  97. ::
  98. self.attr('_my_attribute', lineno=lineno)
  99. """
  100. return nodes.ExtensionAttribute(self.identifier, name, lineno=lineno)
  101. def call_method(
  102. self, name, args=None, kwargs=None, dyn_args=None, dyn_kwargs=None, lineno=None
  103. ):
  104. """Call a method of the extension. This is a shortcut for
  105. :meth:`attr` + :class:`jinja2.nodes.Call`.
  106. """
  107. if args is None:
  108. args = []
  109. if kwargs is None:
  110. kwargs = []
  111. return nodes.Call(
  112. self.attr(name, lineno=lineno),
  113. args,
  114. kwargs,
  115. dyn_args,
  116. dyn_kwargs,
  117. lineno=lineno,
  118. )
  119. @contextfunction
  120. def _gettext_alias(__context, *args, **kwargs):
  121. return __context.call(__context.resolve("gettext"), *args, **kwargs)
  122. def _make_new_gettext(func):
  123. @contextfunction
  124. def gettext(__context, __string, **variables):
  125. rv = __context.call(func, __string)
  126. if __context.eval_ctx.autoescape:
  127. rv = Markup(rv)
  128. # Always treat as a format string, even if there are no
  129. # variables. This makes translation strings more consistent
  130. # and predictable. This requires escaping
  131. return rv % variables
  132. return gettext
  133. def _make_new_ngettext(func):
  134. @contextfunction
  135. def ngettext(__context, __singular, __plural, __num, **variables):
  136. variables.setdefault("num", __num)
  137. rv = __context.call(func, __singular, __plural, __num)
  138. if __context.eval_ctx.autoescape:
  139. rv = Markup(rv)
  140. # Always treat as a format string, see gettext comment above.
  141. return rv % variables
  142. return ngettext
  143. class InternationalizationExtension(Extension):
  144. """This extension adds gettext support to Jinja."""
  145. tags = {"trans"}
  146. # TODO: the i18n extension is currently reevaluating values in a few
  147. # situations. Take this example:
  148. # {% trans count=something() %}{{ count }} foo{% pluralize
  149. # %}{{ count }} fooss{% endtrans %}
  150. # something is called twice here. One time for the gettext value and
  151. # the other time for the n-parameter of the ngettext function.
  152. def __init__(self, environment):
  153. Extension.__init__(self, environment)
  154. environment.globals["_"] = _gettext_alias
  155. environment.extend(
  156. install_gettext_translations=self._install,
  157. install_null_translations=self._install_null,
  158. install_gettext_callables=self._install_callables,
  159. uninstall_gettext_translations=self._uninstall,
  160. extract_translations=self._extract,
  161. newstyle_gettext=False,
  162. )
  163. def _install(self, translations, newstyle=None):
  164. gettext = getattr(translations, "ugettext", None)
  165. if gettext is None:
  166. gettext = translations.gettext
  167. ngettext = getattr(translations, "ungettext", None)
  168. if ngettext is None:
  169. ngettext = translations.ngettext
  170. self._install_callables(gettext, ngettext, newstyle)
  171. def _install_null(self, newstyle=None):
  172. self._install_callables(
  173. lambda x: x, lambda s, p, n: (n != 1 and (p,) or (s,))[0], newstyle
  174. )
  175. def _install_callables(self, gettext, ngettext, newstyle=None):
  176. if newstyle is not None:
  177. self.environment.newstyle_gettext = newstyle
  178. if self.environment.newstyle_gettext:
  179. gettext = _make_new_gettext(gettext)
  180. ngettext = _make_new_ngettext(ngettext)
  181. self.environment.globals.update(gettext=gettext, ngettext=ngettext)
  182. def _uninstall(self, translations):
  183. for key in "gettext", "ngettext":
  184. self.environment.globals.pop(key, None)
  185. def _extract(self, source, gettext_functions=GETTEXT_FUNCTIONS):
  186. if isinstance(source, string_types):
  187. source = self.environment.parse(source)
  188. return extract_from_ast(source, gettext_functions)
  189. def parse(self, parser):
  190. """Parse a translatable tag."""
  191. lineno = next(parser.stream).lineno
  192. num_called_num = False
  193. # find all the variables referenced. Additionally a variable can be
  194. # defined in the body of the trans block too, but this is checked at
  195. # a later state.
  196. plural_expr = None
  197. plural_expr_assignment = None
  198. variables = {}
  199. trimmed = None
  200. while parser.stream.current.type != "block_end":
  201. if variables:
  202. parser.stream.expect("comma")
  203. # skip colon for python compatibility
  204. if parser.stream.skip_if("colon"):
  205. break
  206. name = parser.stream.expect("name")
  207. if name.value in variables:
  208. parser.fail(
  209. "translatable variable %r defined twice." % name.value,
  210. name.lineno,
  211. exc=TemplateAssertionError,
  212. )
  213. # expressions
  214. if parser.stream.current.type == "assign":
  215. next(parser.stream)
  216. variables[name.value] = var = parser.parse_expression()
  217. elif trimmed is None and name.value in ("trimmed", "notrimmed"):
  218. trimmed = name.value == "trimmed"
  219. continue
  220. else:
  221. variables[name.value] = var = nodes.Name(name.value, "load")
  222. if plural_expr is None:
  223. if isinstance(var, nodes.Call):
  224. plural_expr = nodes.Name("_trans", "load")
  225. variables[name.value] = plural_expr
  226. plural_expr_assignment = nodes.Assign(
  227. nodes.Name("_trans", "store"), var
  228. )
  229. else:
  230. plural_expr = var
  231. num_called_num = name.value == "num"
  232. parser.stream.expect("block_end")
  233. plural = None
  234. have_plural = False
  235. referenced = set()
  236. # now parse until endtrans or pluralize
  237. singular_names, singular = self._parse_block(parser, True)
  238. if singular_names:
  239. referenced.update(singular_names)
  240. if plural_expr is None:
  241. plural_expr = nodes.Name(singular_names[0], "load")
  242. num_called_num = singular_names[0] == "num"
  243. # if we have a pluralize block, we parse that too
  244. if parser.stream.current.test("name:pluralize"):
  245. have_plural = True
  246. next(parser.stream)
  247. if parser.stream.current.type != "block_end":
  248. name = parser.stream.expect("name")
  249. if name.value not in variables:
  250. parser.fail(
  251. "unknown variable %r for pluralization" % name.value,
  252. name.lineno,
  253. exc=TemplateAssertionError,
  254. )
  255. plural_expr = variables[name.value]
  256. num_called_num = name.value == "num"
  257. parser.stream.expect("block_end")
  258. plural_names, plural = self._parse_block(parser, False)
  259. next(parser.stream)
  260. referenced.update(plural_names)
  261. else:
  262. next(parser.stream)
  263. # register free names as simple name expressions
  264. for var in referenced:
  265. if var not in variables:
  266. variables[var] = nodes.Name(var, "load")
  267. if not have_plural:
  268. plural_expr = None
  269. elif plural_expr is None:
  270. parser.fail("pluralize without variables", lineno)
  271. if trimmed is None:
  272. trimmed = self.environment.policies["ext.i18n.trimmed"]
  273. if trimmed:
  274. singular = self._trim_whitespace(singular)
  275. if plural:
  276. plural = self._trim_whitespace(plural)
  277. node = self._make_node(
  278. singular,
  279. plural,
  280. variables,
  281. plural_expr,
  282. bool(referenced),
  283. num_called_num and have_plural,
  284. )
  285. node.set_lineno(lineno)
  286. if plural_expr_assignment is not None:
  287. return [plural_expr_assignment, node]
  288. else:
  289. return node
  290. def _trim_whitespace(self, string, _ws_re=_ws_re):
  291. return _ws_re.sub(" ", string.strip())
  292. def _parse_block(self, parser, allow_pluralize):
  293. """Parse until the next block tag with a given name."""
  294. referenced = []
  295. buf = []
  296. while 1:
  297. if parser.stream.current.type == "data":
  298. buf.append(parser.stream.current.value.replace("%", "%%"))
  299. next(parser.stream)
  300. elif parser.stream.current.type == "variable_begin":
  301. next(parser.stream)
  302. name = parser.stream.expect("name").value
  303. referenced.append(name)
  304. buf.append("%%(%s)s" % name)
  305. parser.stream.expect("variable_end")
  306. elif parser.stream.current.type == "block_begin":
  307. next(parser.stream)
  308. if parser.stream.current.test("name:endtrans"):
  309. break
  310. elif parser.stream.current.test("name:pluralize"):
  311. if allow_pluralize:
  312. break
  313. parser.fail(
  314. "a translatable section can have only one pluralize section"
  315. )
  316. parser.fail(
  317. "control structures in translatable sections are not allowed"
  318. )
  319. elif parser.stream.eos:
  320. parser.fail("unclosed translation block")
  321. else:
  322. raise RuntimeError("internal parser error")
  323. return referenced, concat(buf)
  324. def _make_node(
  325. self, singular, plural, variables, plural_expr, vars_referenced, num_called_num
  326. ):
  327. """Generates a useful node from the data provided."""
  328. # no variables referenced? no need to escape for old style
  329. # gettext invocations only if there are vars.
  330. if not vars_referenced and not self.environment.newstyle_gettext:
  331. singular = singular.replace("%%", "%")
  332. if plural:
  333. plural = plural.replace("%%", "%")
  334. # singular only:
  335. if plural_expr is None:
  336. gettext = nodes.Name("gettext", "load")
  337. node = nodes.Call(gettext, [nodes.Const(singular)], [], None, None)
  338. # singular and plural
  339. else:
  340. ngettext = nodes.Name("ngettext", "load")
  341. node = nodes.Call(
  342. ngettext,
  343. [nodes.Const(singular), nodes.Const(plural), plural_expr],
  344. [],
  345. None,
  346. None,
  347. )
  348. # in case newstyle gettext is used, the method is powerful
  349. # enough to handle the variable expansion and autoescape
  350. # handling itself
  351. if self.environment.newstyle_gettext:
  352. for key, value in iteritems(variables):
  353. # the function adds that later anyways in case num was
  354. # called num, so just skip it.
  355. if num_called_num and key == "num":
  356. continue
  357. node.kwargs.append(nodes.Keyword(key, value))
  358. # otherwise do that here
  359. else:
  360. # mark the return value as safe if we are in an
  361. # environment with autoescaping turned on
  362. node = nodes.MarkSafeIfAutoescape(node)
  363. if variables:
  364. node = nodes.Mod(
  365. node,
  366. nodes.Dict(
  367. [
  368. nodes.Pair(nodes.Const(key), value)
  369. for key, value in variables.items()
  370. ]
  371. ),
  372. )
  373. return nodes.Output([node])
  374. class ExprStmtExtension(Extension):
  375. """Adds a `do` tag to Jinja that works like the print statement just
  376. that it doesn't print the return value.
  377. """
  378. tags = set(["do"])
  379. def parse(self, parser):
  380. node = nodes.ExprStmt(lineno=next(parser.stream).lineno)
  381. node.node = parser.parse_tuple()
  382. return node
  383. class LoopControlExtension(Extension):
  384. """Adds break and continue to the template engine."""
  385. tags = set(["break", "continue"])
  386. def parse(self, parser):
  387. token = next(parser.stream)
  388. if token.value == "break":
  389. return nodes.Break(lineno=token.lineno)
  390. return nodes.Continue(lineno=token.lineno)
  391. class WithExtension(Extension):
  392. pass
  393. class AutoEscapeExtension(Extension):
  394. pass
  395. class DebugExtension(Extension):
  396. """A ``{% debug %}`` tag that dumps the available variables,
  397. filters, and tests.
  398. .. code-block:: html+jinja
  399. <pre>{% debug %}</pre>
  400. .. code-block:: text
  401. {'context': {'cycler': <class 'jinja2.utils.Cycler'>,
  402. ...,
  403. 'namespace': <class 'jinja2.utils.Namespace'>},
  404. 'filters': ['abs', 'attr', 'batch', 'capitalize', 'center', 'count', 'd',
  405. ..., 'urlencode', 'urlize', 'wordcount', 'wordwrap', 'xmlattr'],
  406. 'tests': ['!=', '<', '<=', '==', '>', '>=', 'callable', 'defined',
  407. ..., 'odd', 'sameas', 'sequence', 'string', 'undefined', 'upper']}
  408. .. versionadded:: 2.11.0
  409. """
  410. tags = {"debug"}
  411. def parse(self, parser):
  412. lineno = parser.stream.expect("name:debug").lineno
  413. context = ContextReference()
  414. result = self.call_method("_render", [context], lineno=lineno)
  415. return nodes.Output([result], lineno=lineno)
  416. def _render(self, context):
  417. result = {
  418. "context": context.get_all(),
  419. "filters": sorted(self.environment.filters.keys()),
  420. "tests": sorted(self.environment.tests.keys()),
  421. }
  422. # Set the depth since the intent is to show the top few names.
  423. if version_info[:2] >= (3, 4):
  424. return pprint.pformat(result, depth=3, compact=True)
  425. else:
  426. return pprint.pformat(result, depth=3)
  427. def extract_from_ast(node, gettext_functions=GETTEXT_FUNCTIONS, babel_style=True):
  428. """Extract localizable strings from the given template node. Per
  429. default this function returns matches in babel style that means non string
  430. parameters as well as keyword arguments are returned as `None`. This
  431. allows Babel to figure out what you really meant if you are using
  432. gettext functions that allow keyword arguments for placeholder expansion.
  433. If you don't want that behavior set the `babel_style` parameter to `False`
  434. which causes only strings to be returned and parameters are always stored
  435. in tuples. As a consequence invalid gettext calls (calls without a single
  436. string parameter or string parameters after non-string parameters) are
  437. skipped.
  438. This example explains the behavior:
  439. >>> from jinja2 import Environment
  440. >>> env = Environment()
  441. >>> node = env.parse('{{ (_("foo"), _(), ngettext("foo", "bar", 42)) }}')
  442. >>> list(extract_from_ast(node))
  443. [(1, '_', 'foo'), (1, '_', ()), (1, 'ngettext', ('foo', 'bar', None))]
  444. >>> list(extract_from_ast(node, babel_style=False))
  445. [(1, '_', ('foo',)), (1, 'ngettext', ('foo', 'bar'))]
  446. For every string found this function yields a ``(lineno, function,
  447. message)`` tuple, where:
  448. * ``lineno`` is the number of the line on which the string was found,
  449. * ``function`` is the name of the ``gettext`` function used (if the
  450. string was extracted from embedded Python code), and
  451. * ``message`` is the string itself (a ``unicode`` object, or a tuple
  452. of ``unicode`` objects for functions with multiple string arguments).
  453. This extraction function operates on the AST and is because of that unable
  454. to extract any comments. For comment support you have to use the babel
  455. extraction interface or extract comments yourself.
  456. """
  457. for node in node.find_all(nodes.Call):
  458. if (
  459. not isinstance(node.node, nodes.Name)
  460. or node.node.name not in gettext_functions
  461. ):
  462. continue
  463. strings = []
  464. for arg in node.args:
  465. if isinstance(arg, nodes.Const) and isinstance(arg.value, string_types):
  466. strings.append(arg.value)
  467. else:
  468. strings.append(None)
  469. for _ in node.kwargs:
  470. strings.append(None)
  471. if node.dyn_args is not None:
  472. strings.append(None)
  473. if node.dyn_kwargs is not None:
  474. strings.append(None)
  475. if not babel_style:
  476. strings = tuple(x for x in strings if x is not None)
  477. if not strings:
  478. continue
  479. else:
  480. if len(strings) == 1:
  481. strings = strings[0]
  482. else:
  483. strings = tuple(strings)
  484. yield node.lineno, node.node.name, strings
  485. class _CommentFinder(object):
  486. """Helper class to find comments in a token stream. Can only
  487. find comments for gettext calls forwards. Once the comment
  488. from line 4 is found, a comment for line 1 will not return a
  489. usable value.
  490. """
  491. def __init__(self, tokens, comment_tags):
  492. self.tokens = tokens
  493. self.comment_tags = comment_tags
  494. self.offset = 0
  495. self.last_lineno = 0
  496. def find_backwards(self, offset):
  497. try:
  498. for _, token_type, token_value in reversed(
  499. self.tokens[self.offset : offset]
  500. ):
  501. if token_type in ("comment", "linecomment"):
  502. try:
  503. prefix, comment = token_value.split(None, 1)
  504. except ValueError:
  505. continue
  506. if prefix in self.comment_tags:
  507. return [comment.rstrip()]
  508. return []
  509. finally:
  510. self.offset = offset
  511. def find_comments(self, lineno):
  512. if not self.comment_tags or self.last_lineno > lineno:
  513. return []
  514. for idx, (token_lineno, _, _) in enumerate(self.tokens[self.offset :]):
  515. if token_lineno > lineno:
  516. return self.find_backwards(self.offset + idx)
  517. return self.find_backwards(len(self.tokens))
  518. def babel_extract(fileobj, keywords, comment_tags, options):
  519. """Babel extraction method for Jinja templates.
  520. .. versionchanged:: 2.3
  521. Basic support for translation comments was added. If `comment_tags`
  522. is now set to a list of keywords for extraction, the extractor will
  523. try to find the best preceding comment that begins with one of the
  524. keywords. For best results, make sure to not have more than one
  525. gettext call in one line of code and the matching comment in the
  526. same line or the line before.
  527. .. versionchanged:: 2.5.1
  528. The `newstyle_gettext` flag can be set to `True` to enable newstyle
  529. gettext calls.
  530. .. versionchanged:: 2.7
  531. A `silent` option can now be provided. If set to `False` template
  532. syntax errors are propagated instead of being ignored.
  533. :param fileobj: the file-like object the messages should be extracted from
  534. :param keywords: a list of keywords (i.e. function names) that should be
  535. recognized as translation functions
  536. :param comment_tags: a list of translator tags to search for and include
  537. in the results.
  538. :param options: a dictionary of additional options (optional)
  539. :return: an iterator over ``(lineno, funcname, message, comments)`` tuples.
  540. (comments will be empty currently)
  541. """
  542. extensions = set()
  543. for extension in options.get("extensions", "").split(","):
  544. extension = extension.strip()
  545. if not extension:
  546. continue
  547. extensions.add(import_string(extension))
  548. if InternationalizationExtension not in extensions:
  549. extensions.add(InternationalizationExtension)
  550. def getbool(options, key, default=False):
  551. return options.get(key, str(default)).lower() in ("1", "on", "yes", "true")
  552. silent = getbool(options, "silent", True)
  553. environment = Environment(
  554. options.get("block_start_string", BLOCK_START_STRING),
  555. options.get("block_end_string", BLOCK_END_STRING),
  556. options.get("variable_start_string", VARIABLE_START_STRING),
  557. options.get("variable_end_string", VARIABLE_END_STRING),
  558. options.get("comment_start_string", COMMENT_START_STRING),
  559. options.get("comment_end_string", COMMENT_END_STRING),
  560. options.get("line_statement_prefix") or LINE_STATEMENT_PREFIX,
  561. options.get("line_comment_prefix") or LINE_COMMENT_PREFIX,
  562. getbool(options, "trim_blocks", TRIM_BLOCKS),
  563. getbool(options, "lstrip_blocks", LSTRIP_BLOCKS),
  564. NEWLINE_SEQUENCE,
  565. getbool(options, "keep_trailing_newline", KEEP_TRAILING_NEWLINE),
  566. frozenset(extensions),
  567. cache_size=0,
  568. auto_reload=False,
  569. )
  570. if getbool(options, "trimmed"):
  571. environment.policies["ext.i18n.trimmed"] = True
  572. if getbool(options, "newstyle_gettext"):
  573. environment.newstyle_gettext = True
  574. source = fileobj.read().decode(options.get("encoding", "utf-8"))
  575. try:
  576. node = environment.parse(source)
  577. tokens = list(environment.lex(environment.preprocess(source)))
  578. except TemplateSyntaxError:
  579. if not silent:
  580. raise
  581. # skip templates with syntax errors
  582. return
  583. finder = _CommentFinder(tokens, comment_tags)
  584. for lineno, func, message in extract_from_ast(node, keywords):
  585. yield lineno, func, message, finder.find_comments(lineno)
  586. #: nicer import names
  587. i18n = InternationalizationExtension
  588. do = ExprStmtExtension
  589. loopcontrols = LoopControlExtension
  590. with_ = WithExtension
  591. autoescape = AutoEscapeExtension
  592. debug = DebugExtension