compiler.py 65 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783178417851786178717881789179017911792179317941795179617971798179918001801180218031804180518061807180818091810181118121813181418151816181718181819182018211822182318241825182618271828182918301831183218331834183518361837183818391840184118421843
  1. # -*- coding: utf-8 -*-
  2. """Compiles nodes from the parser into Python code."""
  3. from collections import namedtuple
  4. from functools import update_wrapper
  5. from itertools import chain
  6. from keyword import iskeyword as is_python_keyword
  7. from markupsafe import escape
  8. from markupsafe import Markup
  9. from . import nodes
  10. from ._compat import imap
  11. from ._compat import iteritems
  12. from ._compat import izip
  13. from ._compat import NativeStringIO
  14. from ._compat import range_type
  15. from ._compat import string_types
  16. from ._compat import text_type
  17. from .exceptions import TemplateAssertionError
  18. from .idtracking import Symbols
  19. from .idtracking import VAR_LOAD_ALIAS
  20. from .idtracking import VAR_LOAD_PARAMETER
  21. from .idtracking import VAR_LOAD_RESOLVE
  22. from .idtracking import VAR_LOAD_UNDEFINED
  23. from .nodes import EvalContext
  24. from .optimizer import Optimizer
  25. from .utils import concat
  26. from .visitor import NodeVisitor
  27. operators = {
  28. "eq": "==",
  29. "ne": "!=",
  30. "gt": ">",
  31. "gteq": ">=",
  32. "lt": "<",
  33. "lteq": "<=",
  34. "in": "in",
  35. "notin": "not in",
  36. }
  37. # what method to iterate over items do we want to use for dict iteration
  38. # in generated code? on 2.x let's go with iteritems, on 3.x with items
  39. if hasattr(dict, "iteritems"):
  40. dict_item_iter = "iteritems"
  41. else:
  42. dict_item_iter = "items"
  43. code_features = ["division"]
  44. # does this python version support generator stops? (PEP 0479)
  45. try:
  46. exec("from __future__ import generator_stop")
  47. code_features.append("generator_stop")
  48. except SyntaxError:
  49. pass
  50. # does this python version support yield from?
  51. try:
  52. exec("def f(): yield from x()")
  53. except SyntaxError:
  54. supports_yield_from = False
  55. else:
  56. supports_yield_from = True
  57. def optimizeconst(f):
  58. def new_func(self, node, frame, **kwargs):
  59. # Only optimize if the frame is not volatile
  60. if self.optimized and not frame.eval_ctx.volatile:
  61. new_node = self.optimizer.visit(node, frame.eval_ctx)
  62. if new_node != node:
  63. return self.visit(new_node, frame)
  64. return f(self, node, frame, **kwargs)
  65. return update_wrapper(new_func, f)
  66. def generate(
  67. node, environment, name, filename, stream=None, defer_init=False, optimized=True
  68. ):
  69. """Generate the python source for a node tree."""
  70. if not isinstance(node, nodes.Template):
  71. raise TypeError("Can't compile non template nodes")
  72. generator = environment.code_generator_class(
  73. environment, name, filename, stream, defer_init, optimized
  74. )
  75. generator.visit(node)
  76. if stream is None:
  77. return generator.stream.getvalue()
  78. def has_safe_repr(value):
  79. """Does the node have a safe representation?"""
  80. if value is None or value is NotImplemented or value is Ellipsis:
  81. return True
  82. if type(value) in (bool, int, float, complex, range_type, Markup) + string_types:
  83. return True
  84. if type(value) in (tuple, list, set, frozenset):
  85. for item in value:
  86. if not has_safe_repr(item):
  87. return False
  88. return True
  89. elif type(value) is dict:
  90. for key, value in iteritems(value):
  91. if not has_safe_repr(key):
  92. return False
  93. if not has_safe_repr(value):
  94. return False
  95. return True
  96. return False
  97. def find_undeclared(nodes, names):
  98. """Check if the names passed are accessed undeclared. The return value
  99. is a set of all the undeclared names from the sequence of names found.
  100. """
  101. visitor = UndeclaredNameVisitor(names)
  102. try:
  103. for node in nodes:
  104. visitor.visit(node)
  105. except VisitorExit:
  106. pass
  107. return visitor.undeclared
  108. class MacroRef(object):
  109. def __init__(self, node):
  110. self.node = node
  111. self.accesses_caller = False
  112. self.accesses_kwargs = False
  113. self.accesses_varargs = False
  114. class Frame(object):
  115. """Holds compile time information for us."""
  116. def __init__(self, eval_ctx, parent=None, level=None):
  117. self.eval_ctx = eval_ctx
  118. self.symbols = Symbols(parent and parent.symbols or None, level=level)
  119. # a toplevel frame is the root + soft frames such as if conditions.
  120. self.toplevel = False
  121. # the root frame is basically just the outermost frame, so no if
  122. # conditions. This information is used to optimize inheritance
  123. # situations.
  124. self.rootlevel = False
  125. # in some dynamic inheritance situations the compiler needs to add
  126. # write tests around output statements.
  127. self.require_output_check = parent and parent.require_output_check
  128. # inside some tags we are using a buffer rather than yield statements.
  129. # this for example affects {% filter %} or {% macro %}. If a frame
  130. # is buffered this variable points to the name of the list used as
  131. # buffer.
  132. self.buffer = None
  133. # the name of the block we're in, otherwise None.
  134. self.block = parent and parent.block or None
  135. # the parent of this frame
  136. self.parent = parent
  137. if parent is not None:
  138. self.buffer = parent.buffer
  139. def copy(self):
  140. """Create a copy of the current one."""
  141. rv = object.__new__(self.__class__)
  142. rv.__dict__.update(self.__dict__)
  143. rv.symbols = self.symbols.copy()
  144. return rv
  145. def inner(self, isolated=False):
  146. """Return an inner frame."""
  147. if isolated:
  148. return Frame(self.eval_ctx, level=self.symbols.level + 1)
  149. return Frame(self.eval_ctx, self)
  150. def soft(self):
  151. """Return a soft frame. A soft frame may not be modified as
  152. standalone thing as it shares the resources with the frame it
  153. was created of, but it's not a rootlevel frame any longer.
  154. This is only used to implement if-statements.
  155. """
  156. rv = self.copy()
  157. rv.rootlevel = False
  158. return rv
  159. __copy__ = copy
  160. class VisitorExit(RuntimeError):
  161. """Exception used by the `UndeclaredNameVisitor` to signal a stop."""
  162. class DependencyFinderVisitor(NodeVisitor):
  163. """A visitor that collects filter and test calls."""
  164. def __init__(self):
  165. self.filters = set()
  166. self.tests = set()
  167. def visit_Filter(self, node):
  168. self.generic_visit(node)
  169. self.filters.add(node.name)
  170. def visit_Test(self, node):
  171. self.generic_visit(node)
  172. self.tests.add(node.name)
  173. def visit_Block(self, node):
  174. """Stop visiting at blocks."""
  175. class UndeclaredNameVisitor(NodeVisitor):
  176. """A visitor that checks if a name is accessed without being
  177. declared. This is different from the frame visitor as it will
  178. not stop at closure frames.
  179. """
  180. def __init__(self, names):
  181. self.names = set(names)
  182. self.undeclared = set()
  183. def visit_Name(self, node):
  184. if node.ctx == "load" and node.name in self.names:
  185. self.undeclared.add(node.name)
  186. if self.undeclared == self.names:
  187. raise VisitorExit()
  188. else:
  189. self.names.discard(node.name)
  190. def visit_Block(self, node):
  191. """Stop visiting a blocks."""
  192. class CompilerExit(Exception):
  193. """Raised if the compiler encountered a situation where it just
  194. doesn't make sense to further process the code. Any block that
  195. raises such an exception is not further processed.
  196. """
  197. class CodeGenerator(NodeVisitor):
  198. def __init__(
  199. self, environment, name, filename, stream=None, defer_init=False, optimized=True
  200. ):
  201. if stream is None:
  202. stream = NativeStringIO()
  203. self.environment = environment
  204. self.name = name
  205. self.filename = filename
  206. self.stream = stream
  207. self.created_block_context = False
  208. self.defer_init = defer_init
  209. self.optimized = optimized
  210. if optimized:
  211. self.optimizer = Optimizer(environment)
  212. # aliases for imports
  213. self.import_aliases = {}
  214. # a registry for all blocks. Because blocks are moved out
  215. # into the global python scope they are registered here
  216. self.blocks = {}
  217. # the number of extends statements so far
  218. self.extends_so_far = 0
  219. # some templates have a rootlevel extends. In this case we
  220. # can safely assume that we're a child template and do some
  221. # more optimizations.
  222. self.has_known_extends = False
  223. # the current line number
  224. self.code_lineno = 1
  225. # registry of all filters and tests (global, not block local)
  226. self.tests = {}
  227. self.filters = {}
  228. # the debug information
  229. self.debug_info = []
  230. self._write_debug_info = None
  231. # the number of new lines before the next write()
  232. self._new_lines = 0
  233. # the line number of the last written statement
  234. self._last_line = 0
  235. # true if nothing was written so far.
  236. self._first_write = True
  237. # used by the `temporary_identifier` method to get new
  238. # unique, temporary identifier
  239. self._last_identifier = 0
  240. # the current indentation
  241. self._indentation = 0
  242. # Tracks toplevel assignments
  243. self._assign_stack = []
  244. # Tracks parameter definition blocks
  245. self._param_def_block = []
  246. # Tracks the current context.
  247. self._context_reference_stack = ["context"]
  248. # -- Various compilation helpers
  249. def fail(self, msg, lineno):
  250. """Fail with a :exc:`TemplateAssertionError`."""
  251. raise TemplateAssertionError(msg, lineno, self.name, self.filename)
  252. def temporary_identifier(self):
  253. """Get a new unique identifier."""
  254. self._last_identifier += 1
  255. return "t_%d" % self._last_identifier
  256. def buffer(self, frame):
  257. """Enable buffering for the frame from that point onwards."""
  258. frame.buffer = self.temporary_identifier()
  259. self.writeline("%s = []" % frame.buffer)
  260. def return_buffer_contents(self, frame, force_unescaped=False):
  261. """Return the buffer contents of the frame."""
  262. if not force_unescaped:
  263. if frame.eval_ctx.volatile:
  264. self.writeline("if context.eval_ctx.autoescape:")
  265. self.indent()
  266. self.writeline("return Markup(concat(%s))" % frame.buffer)
  267. self.outdent()
  268. self.writeline("else:")
  269. self.indent()
  270. self.writeline("return concat(%s)" % frame.buffer)
  271. self.outdent()
  272. return
  273. elif frame.eval_ctx.autoescape:
  274. self.writeline("return Markup(concat(%s))" % frame.buffer)
  275. return
  276. self.writeline("return concat(%s)" % frame.buffer)
  277. def indent(self):
  278. """Indent by one."""
  279. self._indentation += 1
  280. def outdent(self, step=1):
  281. """Outdent by step."""
  282. self._indentation -= step
  283. def start_write(self, frame, node=None):
  284. """Yield or write into the frame buffer."""
  285. if frame.buffer is None:
  286. self.writeline("yield ", node)
  287. else:
  288. self.writeline("%s.append(" % frame.buffer, node)
  289. def end_write(self, frame):
  290. """End the writing process started by `start_write`."""
  291. if frame.buffer is not None:
  292. self.write(")")
  293. def simple_write(self, s, frame, node=None):
  294. """Simple shortcut for start_write + write + end_write."""
  295. self.start_write(frame, node)
  296. self.write(s)
  297. self.end_write(frame)
  298. def blockvisit(self, nodes, frame):
  299. """Visit a list of nodes as block in a frame. If the current frame
  300. is no buffer a dummy ``if 0: yield None`` is written automatically.
  301. """
  302. try:
  303. self.writeline("pass")
  304. for node in nodes:
  305. self.visit(node, frame)
  306. except CompilerExit:
  307. pass
  308. def write(self, x):
  309. """Write a string into the output stream."""
  310. if self._new_lines:
  311. if not self._first_write:
  312. self.stream.write("\n" * self._new_lines)
  313. self.code_lineno += self._new_lines
  314. if self._write_debug_info is not None:
  315. self.debug_info.append((self._write_debug_info, self.code_lineno))
  316. self._write_debug_info = None
  317. self._first_write = False
  318. self.stream.write(" " * self._indentation)
  319. self._new_lines = 0
  320. self.stream.write(x)
  321. def writeline(self, x, node=None, extra=0):
  322. """Combination of newline and write."""
  323. self.newline(node, extra)
  324. self.write(x)
  325. def newline(self, node=None, extra=0):
  326. """Add one or more newlines before the next write."""
  327. self._new_lines = max(self._new_lines, 1 + extra)
  328. if node is not None and node.lineno != self._last_line:
  329. self._write_debug_info = node.lineno
  330. self._last_line = node.lineno
  331. def signature(self, node, frame, extra_kwargs=None):
  332. """Writes a function call to the stream for the current node.
  333. A leading comma is added automatically. The extra keyword
  334. arguments may not include python keywords otherwise a syntax
  335. error could occur. The extra keyword arguments should be given
  336. as python dict.
  337. """
  338. # if any of the given keyword arguments is a python keyword
  339. # we have to make sure that no invalid call is created.
  340. kwarg_workaround = False
  341. for kwarg in chain((x.key for x in node.kwargs), extra_kwargs or ()):
  342. if is_python_keyword(kwarg):
  343. kwarg_workaround = True
  344. break
  345. for arg in node.args:
  346. self.write(", ")
  347. self.visit(arg, frame)
  348. if not kwarg_workaround:
  349. for kwarg in node.kwargs:
  350. self.write(", ")
  351. self.visit(kwarg, frame)
  352. if extra_kwargs is not None:
  353. for key, value in iteritems(extra_kwargs):
  354. self.write(", %s=%s" % (key, value))
  355. if node.dyn_args:
  356. self.write(", *")
  357. self.visit(node.dyn_args, frame)
  358. if kwarg_workaround:
  359. if node.dyn_kwargs is not None:
  360. self.write(", **dict({")
  361. else:
  362. self.write(", **{")
  363. for kwarg in node.kwargs:
  364. self.write("%r: " % kwarg.key)
  365. self.visit(kwarg.value, frame)
  366. self.write(", ")
  367. if extra_kwargs is not None:
  368. for key, value in iteritems(extra_kwargs):
  369. self.write("%r: %s, " % (key, value))
  370. if node.dyn_kwargs is not None:
  371. self.write("}, **")
  372. self.visit(node.dyn_kwargs, frame)
  373. self.write(")")
  374. else:
  375. self.write("}")
  376. elif node.dyn_kwargs is not None:
  377. self.write(", **")
  378. self.visit(node.dyn_kwargs, frame)
  379. def pull_dependencies(self, nodes):
  380. """Pull all the dependencies."""
  381. visitor = DependencyFinderVisitor()
  382. for node in nodes:
  383. visitor.visit(node)
  384. for dependency in "filters", "tests":
  385. mapping = getattr(self, dependency)
  386. for name in sorted(getattr(visitor, dependency)):
  387. if name not in mapping:
  388. mapping[name] = self.temporary_identifier()
  389. self.writeline(
  390. "%s = environment.%s[%r]" % (mapping[name], dependency, name)
  391. )
  392. def enter_frame(self, frame):
  393. undefs = []
  394. for target, (action, param) in iteritems(frame.symbols.loads):
  395. if action == VAR_LOAD_PARAMETER:
  396. pass
  397. elif action == VAR_LOAD_RESOLVE:
  398. self.writeline("%s = %s(%r)" % (target, self.get_resolve_func(), param))
  399. elif action == VAR_LOAD_ALIAS:
  400. self.writeline("%s = %s" % (target, param))
  401. elif action == VAR_LOAD_UNDEFINED:
  402. undefs.append(target)
  403. else:
  404. raise NotImplementedError("unknown load instruction")
  405. if undefs:
  406. self.writeline("%s = missing" % " = ".join(undefs))
  407. def leave_frame(self, frame, with_python_scope=False):
  408. if not with_python_scope:
  409. undefs = []
  410. for target, _ in iteritems(frame.symbols.loads):
  411. undefs.append(target)
  412. if undefs:
  413. self.writeline("%s = missing" % " = ".join(undefs))
  414. def func(self, name):
  415. if self.environment.is_async:
  416. return "async def %s" % name
  417. return "def %s" % name
  418. def macro_body(self, node, frame):
  419. """Dump the function def of a macro or call block."""
  420. frame = frame.inner()
  421. frame.symbols.analyze_node(node)
  422. macro_ref = MacroRef(node)
  423. explicit_caller = None
  424. skip_special_params = set()
  425. args = []
  426. for idx, arg in enumerate(node.args):
  427. if arg.name == "caller":
  428. explicit_caller = idx
  429. if arg.name in ("kwargs", "varargs"):
  430. skip_special_params.add(arg.name)
  431. args.append(frame.symbols.ref(arg.name))
  432. undeclared = find_undeclared(node.body, ("caller", "kwargs", "varargs"))
  433. if "caller" in undeclared:
  434. # In older Jinja versions there was a bug that allowed caller
  435. # to retain the special behavior even if it was mentioned in
  436. # the argument list. However thankfully this was only really
  437. # working if it was the last argument. So we are explicitly
  438. # checking this now and error out if it is anywhere else in
  439. # the argument list.
  440. if explicit_caller is not None:
  441. try:
  442. node.defaults[explicit_caller - len(node.args)]
  443. except IndexError:
  444. self.fail(
  445. "When defining macros or call blocks the "
  446. 'special "caller" argument must be omitted '
  447. "or be given a default.",
  448. node.lineno,
  449. )
  450. else:
  451. args.append(frame.symbols.declare_parameter("caller"))
  452. macro_ref.accesses_caller = True
  453. if "kwargs" in undeclared and "kwargs" not in skip_special_params:
  454. args.append(frame.symbols.declare_parameter("kwargs"))
  455. macro_ref.accesses_kwargs = True
  456. if "varargs" in undeclared and "varargs" not in skip_special_params:
  457. args.append(frame.symbols.declare_parameter("varargs"))
  458. macro_ref.accesses_varargs = True
  459. # macros are delayed, they never require output checks
  460. frame.require_output_check = False
  461. frame.symbols.analyze_node(node)
  462. self.writeline("%s(%s):" % (self.func("macro"), ", ".join(args)), node)
  463. self.indent()
  464. self.buffer(frame)
  465. self.enter_frame(frame)
  466. self.push_parameter_definitions(frame)
  467. for idx, arg in enumerate(node.args):
  468. ref = frame.symbols.ref(arg.name)
  469. self.writeline("if %s is missing:" % ref)
  470. self.indent()
  471. try:
  472. default = node.defaults[idx - len(node.args)]
  473. except IndexError:
  474. self.writeline(
  475. "%s = undefined(%r, name=%r)"
  476. % (ref, "parameter %r was not provided" % arg.name, arg.name)
  477. )
  478. else:
  479. self.writeline("%s = " % ref)
  480. self.visit(default, frame)
  481. self.mark_parameter_stored(ref)
  482. self.outdent()
  483. self.pop_parameter_definitions()
  484. self.blockvisit(node.body, frame)
  485. self.return_buffer_contents(frame, force_unescaped=True)
  486. self.leave_frame(frame, with_python_scope=True)
  487. self.outdent()
  488. return frame, macro_ref
  489. def macro_def(self, macro_ref, frame):
  490. """Dump the macro definition for the def created by macro_body."""
  491. arg_tuple = ", ".join(repr(x.name) for x in macro_ref.node.args)
  492. name = getattr(macro_ref.node, "name", None)
  493. if len(macro_ref.node.args) == 1:
  494. arg_tuple += ","
  495. self.write(
  496. "Macro(environment, macro, %r, (%s), %r, %r, %r, "
  497. "context.eval_ctx.autoescape)"
  498. % (
  499. name,
  500. arg_tuple,
  501. macro_ref.accesses_kwargs,
  502. macro_ref.accesses_varargs,
  503. macro_ref.accesses_caller,
  504. )
  505. )
  506. def position(self, node):
  507. """Return a human readable position for the node."""
  508. rv = "line %d" % node.lineno
  509. if self.name is not None:
  510. rv += " in " + repr(self.name)
  511. return rv
  512. def dump_local_context(self, frame):
  513. return "{%s}" % ", ".join(
  514. "%r: %s" % (name, target)
  515. for name, target in sorted(iteritems(frame.symbols.dump_stores()))
  516. )
  517. def write_commons(self):
  518. """Writes a common preamble that is used by root and block functions.
  519. Primarily this sets up common local helpers and enforces a generator
  520. through a dead branch.
  521. """
  522. self.writeline("resolve = context.resolve_or_missing")
  523. self.writeline("undefined = environment.undefined")
  524. # always use the standard Undefined class for the implicit else of
  525. # conditional expressions
  526. self.writeline("cond_expr_undefined = Undefined")
  527. self.writeline("if 0: yield None")
  528. def push_parameter_definitions(self, frame):
  529. """Pushes all parameter targets from the given frame into a local
  530. stack that permits tracking of yet to be assigned parameters. In
  531. particular this enables the optimization from `visit_Name` to skip
  532. undefined expressions for parameters in macros as macros can reference
  533. otherwise unbound parameters.
  534. """
  535. self._param_def_block.append(frame.symbols.dump_param_targets())
  536. def pop_parameter_definitions(self):
  537. """Pops the current parameter definitions set."""
  538. self._param_def_block.pop()
  539. def mark_parameter_stored(self, target):
  540. """Marks a parameter in the current parameter definitions as stored.
  541. This will skip the enforced undefined checks.
  542. """
  543. if self._param_def_block:
  544. self._param_def_block[-1].discard(target)
  545. def push_context_reference(self, target):
  546. self._context_reference_stack.append(target)
  547. def pop_context_reference(self):
  548. self._context_reference_stack.pop()
  549. def get_context_ref(self):
  550. return self._context_reference_stack[-1]
  551. def get_resolve_func(self):
  552. target = self._context_reference_stack[-1]
  553. if target == "context":
  554. return "resolve"
  555. return "%s.resolve" % target
  556. def derive_context(self, frame):
  557. return "%s.derived(%s)" % (
  558. self.get_context_ref(),
  559. self.dump_local_context(frame),
  560. )
  561. def parameter_is_undeclared(self, target):
  562. """Checks if a given target is an undeclared parameter."""
  563. if not self._param_def_block:
  564. return False
  565. return target in self._param_def_block[-1]
  566. def push_assign_tracking(self):
  567. """Pushes a new layer for assignment tracking."""
  568. self._assign_stack.append(set())
  569. def pop_assign_tracking(self, frame):
  570. """Pops the topmost level for assignment tracking and updates the
  571. context variables if necessary.
  572. """
  573. vars = self._assign_stack.pop()
  574. if not frame.toplevel or not vars:
  575. return
  576. public_names = [x for x in vars if x[:1] != "_"]
  577. if len(vars) == 1:
  578. name = next(iter(vars))
  579. ref = frame.symbols.ref(name)
  580. self.writeline("context.vars[%r] = %s" % (name, ref))
  581. else:
  582. self.writeline("context.vars.update({")
  583. for idx, name in enumerate(vars):
  584. if idx:
  585. self.write(", ")
  586. ref = frame.symbols.ref(name)
  587. self.write("%r: %s" % (name, ref))
  588. self.write("})")
  589. if public_names:
  590. if len(public_names) == 1:
  591. self.writeline("context.exported_vars.add(%r)" % public_names[0])
  592. else:
  593. self.writeline(
  594. "context.exported_vars.update((%s))"
  595. % ", ".join(imap(repr, public_names))
  596. )
  597. # -- Statement Visitors
  598. def visit_Template(self, node, frame=None):
  599. assert frame is None, "no root frame allowed"
  600. eval_ctx = EvalContext(self.environment, self.name)
  601. from .runtime import exported
  602. self.writeline("from __future__ import %s" % ", ".join(code_features))
  603. self.writeline("from jinja2.runtime import " + ", ".join(exported))
  604. if self.environment.is_async:
  605. self.writeline(
  606. "from jinja2.asyncsupport import auto_await, "
  607. "auto_aiter, AsyncLoopContext"
  608. )
  609. # if we want a deferred initialization we cannot move the
  610. # environment into a local name
  611. envenv = not self.defer_init and ", environment=environment" or ""
  612. # do we have an extends tag at all? If not, we can save some
  613. # overhead by just not processing any inheritance code.
  614. have_extends = node.find(nodes.Extends) is not None
  615. # find all blocks
  616. for block in node.find_all(nodes.Block):
  617. if block.name in self.blocks:
  618. self.fail("block %r defined twice" % block.name, block.lineno)
  619. self.blocks[block.name] = block
  620. # find all imports and import them
  621. for import_ in node.find_all(nodes.ImportedName):
  622. if import_.importname not in self.import_aliases:
  623. imp = import_.importname
  624. self.import_aliases[imp] = alias = self.temporary_identifier()
  625. if "." in imp:
  626. module, obj = imp.rsplit(".", 1)
  627. self.writeline("from %s import %s as %s" % (module, obj, alias))
  628. else:
  629. self.writeline("import %s as %s" % (imp, alias))
  630. # add the load name
  631. self.writeline("name = %r" % self.name)
  632. # generate the root render function.
  633. self.writeline(
  634. "%s(context, missing=missing%s):" % (self.func("root"), envenv), extra=1
  635. )
  636. self.indent()
  637. self.write_commons()
  638. # process the root
  639. frame = Frame(eval_ctx)
  640. if "self" in find_undeclared(node.body, ("self",)):
  641. ref = frame.symbols.declare_parameter("self")
  642. self.writeline("%s = TemplateReference(context)" % ref)
  643. frame.symbols.analyze_node(node)
  644. frame.toplevel = frame.rootlevel = True
  645. frame.require_output_check = have_extends and not self.has_known_extends
  646. if have_extends:
  647. self.writeline("parent_template = None")
  648. self.enter_frame(frame)
  649. self.pull_dependencies(node.body)
  650. self.blockvisit(node.body, frame)
  651. self.leave_frame(frame, with_python_scope=True)
  652. self.outdent()
  653. # make sure that the parent root is called.
  654. if have_extends:
  655. if not self.has_known_extends:
  656. self.indent()
  657. self.writeline("if parent_template is not None:")
  658. self.indent()
  659. if supports_yield_from and not self.environment.is_async:
  660. self.writeline("yield from parent_template.root_render_func(context)")
  661. else:
  662. self.writeline(
  663. "%sfor event in parent_template."
  664. "root_render_func(context):"
  665. % (self.environment.is_async and "async " or "")
  666. )
  667. self.indent()
  668. self.writeline("yield event")
  669. self.outdent()
  670. self.outdent(1 + (not self.has_known_extends))
  671. # at this point we now have the blocks collected and can visit them too.
  672. for name, block in iteritems(self.blocks):
  673. self.writeline(
  674. "%s(context, missing=missing%s):"
  675. % (self.func("block_" + name), envenv),
  676. block,
  677. 1,
  678. )
  679. self.indent()
  680. self.write_commons()
  681. # It's important that we do not make this frame a child of the
  682. # toplevel template. This would cause a variety of
  683. # interesting issues with identifier tracking.
  684. block_frame = Frame(eval_ctx)
  685. undeclared = find_undeclared(block.body, ("self", "super"))
  686. if "self" in undeclared:
  687. ref = block_frame.symbols.declare_parameter("self")
  688. self.writeline("%s = TemplateReference(context)" % ref)
  689. if "super" in undeclared:
  690. ref = block_frame.symbols.declare_parameter("super")
  691. self.writeline("%s = context.super(%r, block_%s)" % (ref, name, name))
  692. block_frame.symbols.analyze_node(block)
  693. block_frame.block = name
  694. self.enter_frame(block_frame)
  695. self.pull_dependencies(block.body)
  696. self.blockvisit(block.body, block_frame)
  697. self.leave_frame(block_frame, with_python_scope=True)
  698. self.outdent()
  699. self.writeline(
  700. "blocks = {%s}" % ", ".join("%r: block_%s" % (x, x) for x in self.blocks),
  701. extra=1,
  702. )
  703. # add a function that returns the debug info
  704. self.writeline(
  705. "debug_info = %r" % "&".join("%s=%s" % x for x in self.debug_info)
  706. )
  707. def visit_Block(self, node, frame):
  708. """Call a block and register it for the template."""
  709. level = 0
  710. if frame.toplevel:
  711. # if we know that we are a child template, there is no need to
  712. # check if we are one
  713. if self.has_known_extends:
  714. return
  715. if self.extends_so_far > 0:
  716. self.writeline("if parent_template is None:")
  717. self.indent()
  718. level += 1
  719. if node.scoped:
  720. context = self.derive_context(frame)
  721. else:
  722. context = self.get_context_ref()
  723. if (
  724. supports_yield_from
  725. and not self.environment.is_async
  726. and frame.buffer is None
  727. ):
  728. self.writeline(
  729. "yield from context.blocks[%r][0](%s)" % (node.name, context), node
  730. )
  731. else:
  732. loop = self.environment.is_async and "async for" or "for"
  733. self.writeline(
  734. "%s event in context.blocks[%r][0](%s):" % (loop, node.name, context),
  735. node,
  736. )
  737. self.indent()
  738. self.simple_write("event", frame)
  739. self.outdent()
  740. self.outdent(level)
  741. def visit_Extends(self, node, frame):
  742. """Calls the extender."""
  743. if not frame.toplevel:
  744. self.fail("cannot use extend from a non top-level scope", node.lineno)
  745. # if the number of extends statements in general is zero so
  746. # far, we don't have to add a check if something extended
  747. # the template before this one.
  748. if self.extends_so_far > 0:
  749. # if we have a known extends we just add a template runtime
  750. # error into the generated code. We could catch that at compile
  751. # time too, but i welcome it not to confuse users by throwing the
  752. # same error at different times just "because we can".
  753. if not self.has_known_extends:
  754. self.writeline("if parent_template is not None:")
  755. self.indent()
  756. self.writeline("raise TemplateRuntimeError(%r)" % "extended multiple times")
  757. # if we have a known extends already we don't need that code here
  758. # as we know that the template execution will end here.
  759. if self.has_known_extends:
  760. raise CompilerExit()
  761. else:
  762. self.outdent()
  763. self.writeline("parent_template = environment.get_template(", node)
  764. self.visit(node.template, frame)
  765. self.write(", %r)" % self.name)
  766. self.writeline(
  767. "for name, parent_block in parent_template.blocks.%s():" % dict_item_iter
  768. )
  769. self.indent()
  770. self.writeline("context.blocks.setdefault(name, []).append(parent_block)")
  771. self.outdent()
  772. # if this extends statement was in the root level we can take
  773. # advantage of that information and simplify the generated code
  774. # in the top level from this point onwards
  775. if frame.rootlevel:
  776. self.has_known_extends = True
  777. # and now we have one more
  778. self.extends_so_far += 1
  779. def visit_Include(self, node, frame):
  780. """Handles includes."""
  781. if node.ignore_missing:
  782. self.writeline("try:")
  783. self.indent()
  784. func_name = "get_or_select_template"
  785. if isinstance(node.template, nodes.Const):
  786. if isinstance(node.template.value, string_types):
  787. func_name = "get_template"
  788. elif isinstance(node.template.value, (tuple, list)):
  789. func_name = "select_template"
  790. elif isinstance(node.template, (nodes.Tuple, nodes.List)):
  791. func_name = "select_template"
  792. self.writeline("template = environment.%s(" % func_name, node)
  793. self.visit(node.template, frame)
  794. self.write(", %r)" % self.name)
  795. if node.ignore_missing:
  796. self.outdent()
  797. self.writeline("except TemplateNotFound:")
  798. self.indent()
  799. self.writeline("pass")
  800. self.outdent()
  801. self.writeline("else:")
  802. self.indent()
  803. skip_event_yield = False
  804. if node.with_context:
  805. loop = self.environment.is_async and "async for" or "for"
  806. self.writeline(
  807. "%s event in template.root_render_func("
  808. "template.new_context(context.get_all(), True, "
  809. "%s)):" % (loop, self.dump_local_context(frame))
  810. )
  811. elif self.environment.is_async:
  812. self.writeline(
  813. "for event in (await "
  814. "template._get_default_module_async())"
  815. "._body_stream:"
  816. )
  817. else:
  818. if supports_yield_from:
  819. self.writeline("yield from template._get_default_module()._body_stream")
  820. skip_event_yield = True
  821. else:
  822. self.writeline(
  823. "for event in template._get_default_module()._body_stream:"
  824. )
  825. if not skip_event_yield:
  826. self.indent()
  827. self.simple_write("event", frame)
  828. self.outdent()
  829. if node.ignore_missing:
  830. self.outdent()
  831. def visit_Import(self, node, frame):
  832. """Visit regular imports."""
  833. self.writeline("%s = " % frame.symbols.ref(node.target), node)
  834. if frame.toplevel:
  835. self.write("context.vars[%r] = " % node.target)
  836. if self.environment.is_async:
  837. self.write("await ")
  838. self.write("environment.get_template(")
  839. self.visit(node.template, frame)
  840. self.write(", %r)." % self.name)
  841. if node.with_context:
  842. self.write(
  843. "make_module%s(context.get_all(), True, %s)"
  844. % (
  845. self.environment.is_async and "_async" or "",
  846. self.dump_local_context(frame),
  847. )
  848. )
  849. elif self.environment.is_async:
  850. self.write("_get_default_module_async()")
  851. else:
  852. self.write("_get_default_module()")
  853. if frame.toplevel and not node.target.startswith("_"):
  854. self.writeline("context.exported_vars.discard(%r)" % node.target)
  855. def visit_FromImport(self, node, frame):
  856. """Visit named imports."""
  857. self.newline(node)
  858. self.write(
  859. "included_template = %senvironment.get_template("
  860. % (self.environment.is_async and "await " or "")
  861. )
  862. self.visit(node.template, frame)
  863. self.write(", %r)." % self.name)
  864. if node.with_context:
  865. self.write(
  866. "make_module%s(context.get_all(), True, %s)"
  867. % (
  868. self.environment.is_async and "_async" or "",
  869. self.dump_local_context(frame),
  870. )
  871. )
  872. elif self.environment.is_async:
  873. self.write("_get_default_module_async()")
  874. else:
  875. self.write("_get_default_module()")
  876. var_names = []
  877. discarded_names = []
  878. for name in node.names:
  879. if isinstance(name, tuple):
  880. name, alias = name
  881. else:
  882. alias = name
  883. self.writeline(
  884. "%s = getattr(included_template, "
  885. "%r, missing)" % (frame.symbols.ref(alias), name)
  886. )
  887. self.writeline("if %s is missing:" % frame.symbols.ref(alias))
  888. self.indent()
  889. self.writeline(
  890. "%s = undefined(%r %% "
  891. "included_template.__name__, "
  892. "name=%r)"
  893. % (
  894. frame.symbols.ref(alias),
  895. "the template %%r (imported on %s) does "
  896. "not export the requested name %s"
  897. % (self.position(node), repr(name)),
  898. name,
  899. )
  900. )
  901. self.outdent()
  902. if frame.toplevel:
  903. var_names.append(alias)
  904. if not alias.startswith("_"):
  905. discarded_names.append(alias)
  906. if var_names:
  907. if len(var_names) == 1:
  908. name = var_names[0]
  909. self.writeline(
  910. "context.vars[%r] = %s" % (name, frame.symbols.ref(name))
  911. )
  912. else:
  913. self.writeline(
  914. "context.vars.update({%s})"
  915. % ", ".join(
  916. "%r: %s" % (name, frame.symbols.ref(name)) for name in var_names
  917. )
  918. )
  919. if discarded_names:
  920. if len(discarded_names) == 1:
  921. self.writeline("context.exported_vars.discard(%r)" % discarded_names[0])
  922. else:
  923. self.writeline(
  924. "context.exported_vars.difference_"
  925. "update((%s))" % ", ".join(imap(repr, discarded_names))
  926. )
  927. def visit_For(self, node, frame):
  928. loop_frame = frame.inner()
  929. test_frame = frame.inner()
  930. else_frame = frame.inner()
  931. # try to figure out if we have an extended loop. An extended loop
  932. # is necessary if the loop is in recursive mode if the special loop
  933. # variable is accessed in the body.
  934. extended_loop = node.recursive or "loop" in find_undeclared(
  935. node.iter_child_nodes(only=("body",)), ("loop",)
  936. )
  937. loop_ref = None
  938. if extended_loop:
  939. loop_ref = loop_frame.symbols.declare_parameter("loop")
  940. loop_frame.symbols.analyze_node(node, for_branch="body")
  941. if node.else_:
  942. else_frame.symbols.analyze_node(node, for_branch="else")
  943. if node.test:
  944. loop_filter_func = self.temporary_identifier()
  945. test_frame.symbols.analyze_node(node, for_branch="test")
  946. self.writeline("%s(fiter):" % self.func(loop_filter_func), node.test)
  947. self.indent()
  948. self.enter_frame(test_frame)
  949. self.writeline(self.environment.is_async and "async for " or "for ")
  950. self.visit(node.target, loop_frame)
  951. self.write(" in ")
  952. self.write(self.environment.is_async and "auto_aiter(fiter)" or "fiter")
  953. self.write(":")
  954. self.indent()
  955. self.writeline("if ", node.test)
  956. self.visit(node.test, test_frame)
  957. self.write(":")
  958. self.indent()
  959. self.writeline("yield ")
  960. self.visit(node.target, loop_frame)
  961. self.outdent(3)
  962. self.leave_frame(test_frame, with_python_scope=True)
  963. # if we don't have an recursive loop we have to find the shadowed
  964. # variables at that point. Because loops can be nested but the loop
  965. # variable is a special one we have to enforce aliasing for it.
  966. if node.recursive:
  967. self.writeline(
  968. "%s(reciter, loop_render_func, depth=0):" % self.func("loop"), node
  969. )
  970. self.indent()
  971. self.buffer(loop_frame)
  972. # Use the same buffer for the else frame
  973. else_frame.buffer = loop_frame.buffer
  974. # make sure the loop variable is a special one and raise a template
  975. # assertion error if a loop tries to write to loop
  976. if extended_loop:
  977. self.writeline("%s = missing" % loop_ref)
  978. for name in node.find_all(nodes.Name):
  979. if name.ctx == "store" and name.name == "loop":
  980. self.fail(
  981. "Can't assign to special loop variable in for-loop target",
  982. name.lineno,
  983. )
  984. if node.else_:
  985. iteration_indicator = self.temporary_identifier()
  986. self.writeline("%s = 1" % iteration_indicator)
  987. self.writeline(self.environment.is_async and "async for " or "for ", node)
  988. self.visit(node.target, loop_frame)
  989. if extended_loop:
  990. if self.environment.is_async:
  991. self.write(", %s in AsyncLoopContext(" % loop_ref)
  992. else:
  993. self.write(", %s in LoopContext(" % loop_ref)
  994. else:
  995. self.write(" in ")
  996. if node.test:
  997. self.write("%s(" % loop_filter_func)
  998. if node.recursive:
  999. self.write("reciter")
  1000. else:
  1001. if self.environment.is_async and not extended_loop:
  1002. self.write("auto_aiter(")
  1003. self.visit(node.iter, frame)
  1004. if self.environment.is_async and not extended_loop:
  1005. self.write(")")
  1006. if node.test:
  1007. self.write(")")
  1008. if node.recursive:
  1009. self.write(", undefined, loop_render_func, depth):")
  1010. else:
  1011. self.write(extended_loop and ", undefined):" or ":")
  1012. self.indent()
  1013. self.enter_frame(loop_frame)
  1014. self.blockvisit(node.body, loop_frame)
  1015. if node.else_:
  1016. self.writeline("%s = 0" % iteration_indicator)
  1017. self.outdent()
  1018. self.leave_frame(
  1019. loop_frame, with_python_scope=node.recursive and not node.else_
  1020. )
  1021. if node.else_:
  1022. self.writeline("if %s:" % iteration_indicator)
  1023. self.indent()
  1024. self.enter_frame(else_frame)
  1025. self.blockvisit(node.else_, else_frame)
  1026. self.leave_frame(else_frame)
  1027. self.outdent()
  1028. # if the node was recursive we have to return the buffer contents
  1029. # and start the iteration code
  1030. if node.recursive:
  1031. self.return_buffer_contents(loop_frame)
  1032. self.outdent()
  1033. self.start_write(frame, node)
  1034. if self.environment.is_async:
  1035. self.write("await ")
  1036. self.write("loop(")
  1037. if self.environment.is_async:
  1038. self.write("auto_aiter(")
  1039. self.visit(node.iter, frame)
  1040. if self.environment.is_async:
  1041. self.write(")")
  1042. self.write(", loop)")
  1043. self.end_write(frame)
  1044. def visit_If(self, node, frame):
  1045. if_frame = frame.soft()
  1046. self.writeline("if ", node)
  1047. self.visit(node.test, if_frame)
  1048. self.write(":")
  1049. self.indent()
  1050. self.blockvisit(node.body, if_frame)
  1051. self.outdent()
  1052. for elif_ in node.elif_:
  1053. self.writeline("elif ", elif_)
  1054. self.visit(elif_.test, if_frame)
  1055. self.write(":")
  1056. self.indent()
  1057. self.blockvisit(elif_.body, if_frame)
  1058. self.outdent()
  1059. if node.else_:
  1060. self.writeline("else:")
  1061. self.indent()
  1062. self.blockvisit(node.else_, if_frame)
  1063. self.outdent()
  1064. def visit_Macro(self, node, frame):
  1065. macro_frame, macro_ref = self.macro_body(node, frame)
  1066. self.newline()
  1067. if frame.toplevel:
  1068. if not node.name.startswith("_"):
  1069. self.write("context.exported_vars.add(%r)" % node.name)
  1070. self.writeline("context.vars[%r] = " % node.name)
  1071. self.write("%s = " % frame.symbols.ref(node.name))
  1072. self.macro_def(macro_ref, macro_frame)
  1073. def visit_CallBlock(self, node, frame):
  1074. call_frame, macro_ref = self.macro_body(node, frame)
  1075. self.writeline("caller = ")
  1076. self.macro_def(macro_ref, call_frame)
  1077. self.start_write(frame, node)
  1078. self.visit_Call(node.call, frame, forward_caller=True)
  1079. self.end_write(frame)
  1080. def visit_FilterBlock(self, node, frame):
  1081. filter_frame = frame.inner()
  1082. filter_frame.symbols.analyze_node(node)
  1083. self.enter_frame(filter_frame)
  1084. self.buffer(filter_frame)
  1085. self.blockvisit(node.body, filter_frame)
  1086. self.start_write(frame, node)
  1087. self.visit_Filter(node.filter, filter_frame)
  1088. self.end_write(frame)
  1089. self.leave_frame(filter_frame)
  1090. def visit_With(self, node, frame):
  1091. with_frame = frame.inner()
  1092. with_frame.symbols.analyze_node(node)
  1093. self.enter_frame(with_frame)
  1094. for target, expr in izip(node.targets, node.values):
  1095. self.newline()
  1096. self.visit(target, with_frame)
  1097. self.write(" = ")
  1098. self.visit(expr, frame)
  1099. self.blockvisit(node.body, with_frame)
  1100. self.leave_frame(with_frame)
  1101. def visit_ExprStmt(self, node, frame):
  1102. self.newline(node)
  1103. self.visit(node.node, frame)
  1104. _FinalizeInfo = namedtuple("_FinalizeInfo", ("const", "src"))
  1105. #: The default finalize function if the environment isn't configured
  1106. #: with one. Or if the environment has one, this is called on that
  1107. #: function's output for constants.
  1108. _default_finalize = text_type
  1109. _finalize = None
  1110. def _make_finalize(self):
  1111. """Build the finalize function to be used on constants and at
  1112. runtime. Cached so it's only created once for all output nodes.
  1113. Returns a ``namedtuple`` with the following attributes:
  1114. ``const``
  1115. A function to finalize constant data at compile time.
  1116. ``src``
  1117. Source code to output around nodes to be evaluated at
  1118. runtime.
  1119. """
  1120. if self._finalize is not None:
  1121. return self._finalize
  1122. finalize = default = self._default_finalize
  1123. src = None
  1124. if self.environment.finalize:
  1125. src = "environment.finalize("
  1126. env_finalize = self.environment.finalize
  1127. def finalize(value):
  1128. return default(env_finalize(value))
  1129. if getattr(env_finalize, "contextfunction", False) is True:
  1130. src += "context, "
  1131. finalize = None # noqa: F811
  1132. elif getattr(env_finalize, "evalcontextfunction", False) is True:
  1133. src += "context.eval_ctx, "
  1134. finalize = None
  1135. elif getattr(env_finalize, "environmentfunction", False) is True:
  1136. src += "environment, "
  1137. def finalize(value):
  1138. return default(env_finalize(self.environment, value))
  1139. self._finalize = self._FinalizeInfo(finalize, src)
  1140. return self._finalize
  1141. def _output_const_repr(self, group):
  1142. """Given a group of constant values converted from ``Output``
  1143. child nodes, produce a string to write to the template module
  1144. source.
  1145. """
  1146. return repr(concat(group))
  1147. def _output_child_to_const(self, node, frame, finalize):
  1148. """Try to optimize a child of an ``Output`` node by trying to
  1149. convert it to constant, finalized data at compile time.
  1150. If :exc:`Impossible` is raised, the node is not constant and
  1151. will be evaluated at runtime. Any other exception will also be
  1152. evaluated at runtime for easier debugging.
  1153. """
  1154. const = node.as_const(frame.eval_ctx)
  1155. if frame.eval_ctx.autoescape:
  1156. const = escape(const)
  1157. # Template data doesn't go through finalize.
  1158. if isinstance(node, nodes.TemplateData):
  1159. return text_type(const)
  1160. return finalize.const(const)
  1161. def _output_child_pre(self, node, frame, finalize):
  1162. """Output extra source code before visiting a child of an
  1163. ``Output`` node.
  1164. """
  1165. if frame.eval_ctx.volatile:
  1166. self.write("(escape if context.eval_ctx.autoescape else to_string)(")
  1167. elif frame.eval_ctx.autoescape:
  1168. self.write("escape(")
  1169. else:
  1170. self.write("to_string(")
  1171. if finalize.src is not None:
  1172. self.write(finalize.src)
  1173. def _output_child_post(self, node, frame, finalize):
  1174. """Output extra source code after visiting a child of an
  1175. ``Output`` node.
  1176. """
  1177. self.write(")")
  1178. if finalize.src is not None:
  1179. self.write(")")
  1180. def visit_Output(self, node, frame):
  1181. # If an extends is active, don't render outside a block.
  1182. if frame.require_output_check:
  1183. # A top-level extends is known to exist at compile time.
  1184. if self.has_known_extends:
  1185. return
  1186. self.writeline("if parent_template is None:")
  1187. self.indent()
  1188. finalize = self._make_finalize()
  1189. body = []
  1190. # Evaluate constants at compile time if possible. Each item in
  1191. # body will be either a list of static data or a node to be
  1192. # evaluated at runtime.
  1193. for child in node.nodes:
  1194. try:
  1195. if not (
  1196. # If the finalize function requires runtime context,
  1197. # constants can't be evaluated at compile time.
  1198. finalize.const
  1199. # Unless it's basic template data that won't be
  1200. # finalized anyway.
  1201. or isinstance(child, nodes.TemplateData)
  1202. ):
  1203. raise nodes.Impossible()
  1204. const = self._output_child_to_const(child, frame, finalize)
  1205. except (nodes.Impossible, Exception):
  1206. # The node was not constant and needs to be evaluated at
  1207. # runtime. Or another error was raised, which is easier
  1208. # to debug at runtime.
  1209. body.append(child)
  1210. continue
  1211. if body and isinstance(body[-1], list):
  1212. body[-1].append(const)
  1213. else:
  1214. body.append([const])
  1215. if frame.buffer is not None:
  1216. if len(body) == 1:
  1217. self.writeline("%s.append(" % frame.buffer)
  1218. else:
  1219. self.writeline("%s.extend((" % frame.buffer)
  1220. self.indent()
  1221. for item in body:
  1222. if isinstance(item, list):
  1223. # A group of constant data to join and output.
  1224. val = self._output_const_repr(item)
  1225. if frame.buffer is None:
  1226. self.writeline("yield " + val)
  1227. else:
  1228. self.writeline(val + ",")
  1229. else:
  1230. if frame.buffer is None:
  1231. self.writeline("yield ", item)
  1232. else:
  1233. self.newline(item)
  1234. # A node to be evaluated at runtime.
  1235. self._output_child_pre(item, frame, finalize)
  1236. self.visit(item, frame)
  1237. self._output_child_post(item, frame, finalize)
  1238. if frame.buffer is not None:
  1239. self.write(",")
  1240. if frame.buffer is not None:
  1241. self.outdent()
  1242. self.writeline(")" if len(body) == 1 else "))")
  1243. if frame.require_output_check:
  1244. self.outdent()
  1245. def visit_Assign(self, node, frame):
  1246. self.push_assign_tracking()
  1247. self.newline(node)
  1248. self.visit(node.target, frame)
  1249. self.write(" = ")
  1250. self.visit(node.node, frame)
  1251. self.pop_assign_tracking(frame)
  1252. def visit_AssignBlock(self, node, frame):
  1253. self.push_assign_tracking()
  1254. block_frame = frame.inner()
  1255. # This is a special case. Since a set block always captures we
  1256. # will disable output checks. This way one can use set blocks
  1257. # toplevel even in extended templates.
  1258. block_frame.require_output_check = False
  1259. block_frame.symbols.analyze_node(node)
  1260. self.enter_frame(block_frame)
  1261. self.buffer(block_frame)
  1262. self.blockvisit(node.body, block_frame)
  1263. self.newline(node)
  1264. self.visit(node.target, frame)
  1265. self.write(" = (Markup if context.eval_ctx.autoescape else identity)(")
  1266. if node.filter is not None:
  1267. self.visit_Filter(node.filter, block_frame)
  1268. else:
  1269. self.write("concat(%s)" % block_frame.buffer)
  1270. self.write(")")
  1271. self.pop_assign_tracking(frame)
  1272. self.leave_frame(block_frame)
  1273. # -- Expression Visitors
  1274. def visit_Name(self, node, frame):
  1275. if node.ctx == "store" and frame.toplevel:
  1276. if self._assign_stack:
  1277. self._assign_stack[-1].add(node.name)
  1278. ref = frame.symbols.ref(node.name)
  1279. # If we are looking up a variable we might have to deal with the
  1280. # case where it's undefined. We can skip that case if the load
  1281. # instruction indicates a parameter which are always defined.
  1282. if node.ctx == "load":
  1283. load = frame.symbols.find_load(ref)
  1284. if not (
  1285. load is not None
  1286. and load[0] == VAR_LOAD_PARAMETER
  1287. and not self.parameter_is_undeclared(ref)
  1288. ):
  1289. self.write(
  1290. "(undefined(name=%r) if %s is missing else %s)"
  1291. % (node.name, ref, ref)
  1292. )
  1293. return
  1294. self.write(ref)
  1295. def visit_NSRef(self, node, frame):
  1296. # NSRefs can only be used to store values; since they use the normal
  1297. # `foo.bar` notation they will be parsed as a normal attribute access
  1298. # when used anywhere but in a `set` context
  1299. ref = frame.symbols.ref(node.name)
  1300. self.writeline("if not isinstance(%s, Namespace):" % ref)
  1301. self.indent()
  1302. self.writeline(
  1303. "raise TemplateRuntimeError(%r)"
  1304. % "cannot assign attribute on non-namespace object"
  1305. )
  1306. self.outdent()
  1307. self.writeline("%s[%r]" % (ref, node.attr))
  1308. def visit_Const(self, node, frame):
  1309. val = node.as_const(frame.eval_ctx)
  1310. if isinstance(val, float):
  1311. self.write(str(val))
  1312. else:
  1313. self.write(repr(val))
  1314. def visit_TemplateData(self, node, frame):
  1315. try:
  1316. self.write(repr(node.as_const(frame.eval_ctx)))
  1317. except nodes.Impossible:
  1318. self.write(
  1319. "(Markup if context.eval_ctx.autoescape else identity)(%r)" % node.data
  1320. )
  1321. def visit_Tuple(self, node, frame):
  1322. self.write("(")
  1323. idx = -1
  1324. for idx, item in enumerate(node.items):
  1325. if idx:
  1326. self.write(", ")
  1327. self.visit(item, frame)
  1328. self.write(idx == 0 and ",)" or ")")
  1329. def visit_List(self, node, frame):
  1330. self.write("[")
  1331. for idx, item in enumerate(node.items):
  1332. if idx:
  1333. self.write(", ")
  1334. self.visit(item, frame)
  1335. self.write("]")
  1336. def visit_Dict(self, node, frame):
  1337. self.write("{")
  1338. for idx, item in enumerate(node.items):
  1339. if idx:
  1340. self.write(", ")
  1341. self.visit(item.key, frame)
  1342. self.write(": ")
  1343. self.visit(item.value, frame)
  1344. self.write("}")
  1345. def binop(operator, interceptable=True): # noqa: B902
  1346. @optimizeconst
  1347. def visitor(self, node, frame):
  1348. if (
  1349. self.environment.sandboxed
  1350. and operator in self.environment.intercepted_binops
  1351. ):
  1352. self.write("environment.call_binop(context, %r, " % operator)
  1353. self.visit(node.left, frame)
  1354. self.write(", ")
  1355. self.visit(node.right, frame)
  1356. else:
  1357. self.write("(")
  1358. self.visit(node.left, frame)
  1359. self.write(" %s " % operator)
  1360. self.visit(node.right, frame)
  1361. self.write(")")
  1362. return visitor
  1363. def uaop(operator, interceptable=True): # noqa: B902
  1364. @optimizeconst
  1365. def visitor(self, node, frame):
  1366. if (
  1367. self.environment.sandboxed
  1368. and operator in self.environment.intercepted_unops
  1369. ):
  1370. self.write("environment.call_unop(context, %r, " % operator)
  1371. self.visit(node.node, frame)
  1372. else:
  1373. self.write("(" + operator)
  1374. self.visit(node.node, frame)
  1375. self.write(")")
  1376. return visitor
  1377. visit_Add = binop("+")
  1378. visit_Sub = binop("-")
  1379. visit_Mul = binop("*")
  1380. visit_Div = binop("/")
  1381. visit_FloorDiv = binop("//")
  1382. visit_Pow = binop("**")
  1383. visit_Mod = binop("%")
  1384. visit_And = binop("and", interceptable=False)
  1385. visit_Or = binop("or", interceptable=False)
  1386. visit_Pos = uaop("+")
  1387. visit_Neg = uaop("-")
  1388. visit_Not = uaop("not ", interceptable=False)
  1389. del binop, uaop
  1390. @optimizeconst
  1391. def visit_Concat(self, node, frame):
  1392. if frame.eval_ctx.volatile:
  1393. func_name = "(context.eval_ctx.volatile and markup_join or unicode_join)"
  1394. elif frame.eval_ctx.autoescape:
  1395. func_name = "markup_join"
  1396. else:
  1397. func_name = "unicode_join"
  1398. self.write("%s((" % func_name)
  1399. for arg in node.nodes:
  1400. self.visit(arg, frame)
  1401. self.write(", ")
  1402. self.write("))")
  1403. @optimizeconst
  1404. def visit_Compare(self, node, frame):
  1405. self.write("(")
  1406. self.visit(node.expr, frame)
  1407. for op in node.ops:
  1408. self.visit(op, frame)
  1409. self.write(")")
  1410. def visit_Operand(self, node, frame):
  1411. self.write(" %s " % operators[node.op])
  1412. self.visit(node.expr, frame)
  1413. @optimizeconst
  1414. def visit_Getattr(self, node, frame):
  1415. if self.environment.is_async:
  1416. self.write("(await auto_await(")
  1417. self.write("environment.getattr(")
  1418. self.visit(node.node, frame)
  1419. self.write(", %r)" % node.attr)
  1420. if self.environment.is_async:
  1421. self.write("))")
  1422. @optimizeconst
  1423. def visit_Getitem(self, node, frame):
  1424. # slices bypass the environment getitem method.
  1425. if isinstance(node.arg, nodes.Slice):
  1426. self.visit(node.node, frame)
  1427. self.write("[")
  1428. self.visit(node.arg, frame)
  1429. self.write("]")
  1430. else:
  1431. if self.environment.is_async:
  1432. self.write("(await auto_await(")
  1433. self.write("environment.getitem(")
  1434. self.visit(node.node, frame)
  1435. self.write(", ")
  1436. self.visit(node.arg, frame)
  1437. self.write(")")
  1438. if self.environment.is_async:
  1439. self.write("))")
  1440. def visit_Slice(self, node, frame):
  1441. if node.start is not None:
  1442. self.visit(node.start, frame)
  1443. self.write(":")
  1444. if node.stop is not None:
  1445. self.visit(node.stop, frame)
  1446. if node.step is not None:
  1447. self.write(":")
  1448. self.visit(node.step, frame)
  1449. @optimizeconst
  1450. def visit_Filter(self, node, frame):
  1451. if self.environment.is_async:
  1452. self.write("await auto_await(")
  1453. self.write(self.filters[node.name] + "(")
  1454. func = self.environment.filters.get(node.name)
  1455. if func is None:
  1456. self.fail("no filter named %r" % node.name, node.lineno)
  1457. if getattr(func, "contextfilter", False) is True:
  1458. self.write("context, ")
  1459. elif getattr(func, "evalcontextfilter", False) is True:
  1460. self.write("context.eval_ctx, ")
  1461. elif getattr(func, "environmentfilter", False) is True:
  1462. self.write("environment, ")
  1463. # if the filter node is None we are inside a filter block
  1464. # and want to write to the current buffer
  1465. if node.node is not None:
  1466. self.visit(node.node, frame)
  1467. elif frame.eval_ctx.volatile:
  1468. self.write(
  1469. "(context.eval_ctx.autoescape and"
  1470. " Markup(concat(%s)) or concat(%s))" % (frame.buffer, frame.buffer)
  1471. )
  1472. elif frame.eval_ctx.autoescape:
  1473. self.write("Markup(concat(%s))" % frame.buffer)
  1474. else:
  1475. self.write("concat(%s)" % frame.buffer)
  1476. self.signature(node, frame)
  1477. self.write(")")
  1478. if self.environment.is_async:
  1479. self.write(")")
  1480. @optimizeconst
  1481. def visit_Test(self, node, frame):
  1482. self.write(self.tests[node.name] + "(")
  1483. if node.name not in self.environment.tests:
  1484. self.fail("no test named %r" % node.name, node.lineno)
  1485. self.visit(node.node, frame)
  1486. self.signature(node, frame)
  1487. self.write(")")
  1488. @optimizeconst
  1489. def visit_CondExpr(self, node, frame):
  1490. def write_expr2():
  1491. if node.expr2 is not None:
  1492. return self.visit(node.expr2, frame)
  1493. self.write(
  1494. "cond_expr_undefined(%r)"
  1495. % (
  1496. "the inline if-"
  1497. "expression on %s evaluated to false and "
  1498. "no else section was defined." % self.position(node)
  1499. )
  1500. )
  1501. self.write("(")
  1502. self.visit(node.expr1, frame)
  1503. self.write(" if ")
  1504. self.visit(node.test, frame)
  1505. self.write(" else ")
  1506. write_expr2()
  1507. self.write(")")
  1508. @optimizeconst
  1509. def visit_Call(self, node, frame, forward_caller=False):
  1510. if self.environment.is_async:
  1511. self.write("await auto_await(")
  1512. if self.environment.sandboxed:
  1513. self.write("environment.call(context, ")
  1514. else:
  1515. self.write("context.call(")
  1516. self.visit(node.node, frame)
  1517. extra_kwargs = forward_caller and {"caller": "caller"} or None
  1518. self.signature(node, frame, extra_kwargs)
  1519. self.write(")")
  1520. if self.environment.is_async:
  1521. self.write(")")
  1522. def visit_Keyword(self, node, frame):
  1523. self.write(node.key + "=")
  1524. self.visit(node.value, frame)
  1525. # -- Unused nodes for extensions
  1526. def visit_MarkSafe(self, node, frame):
  1527. self.write("Markup(")
  1528. self.visit(node.expr, frame)
  1529. self.write(")")
  1530. def visit_MarkSafeIfAutoescape(self, node, frame):
  1531. self.write("(context.eval_ctx.autoescape and Markup or identity)(")
  1532. self.visit(node.expr, frame)
  1533. self.write(")")
  1534. def visit_EnvironmentAttribute(self, node, frame):
  1535. self.write("environment." + node.name)
  1536. def visit_ExtensionAttribute(self, node, frame):
  1537. self.write("environment.extensions[%r].%s" % (node.identifier, node.name))
  1538. def visit_ImportedName(self, node, frame):
  1539. self.write(self.import_aliases[node.importname])
  1540. def visit_InternalName(self, node, frame):
  1541. self.write(node.name)
  1542. def visit_ContextReference(self, node, frame):
  1543. self.write("context")
  1544. def visit_DerivedContextReference(self, node, frame):
  1545. self.write(self.derive_context(frame))
  1546. def visit_Continue(self, node, frame):
  1547. self.writeline("continue", node)
  1548. def visit_Break(self, node, frame):
  1549. self.writeline("break", node)
  1550. def visit_Scope(self, node, frame):
  1551. scope_frame = frame.inner()
  1552. scope_frame.symbols.analyze_node(node)
  1553. self.enter_frame(scope_frame)
  1554. self.blockvisit(node.body, scope_frame)
  1555. self.leave_frame(scope_frame)
  1556. def visit_OverlayScope(self, node, frame):
  1557. ctx = self.temporary_identifier()
  1558. self.writeline("%s = %s" % (ctx, self.derive_context(frame)))
  1559. self.writeline("%s.vars = " % ctx)
  1560. self.visit(node.context, frame)
  1561. self.push_context_reference(ctx)
  1562. scope_frame = frame.inner(isolated=True)
  1563. scope_frame.symbols.analyze_node(node)
  1564. self.enter_frame(scope_frame)
  1565. self.blockvisit(node.body, scope_frame)
  1566. self.leave_frame(scope_frame)
  1567. self.pop_context_reference()
  1568. def visit_EvalContextModifier(self, node, frame):
  1569. for keyword in node.options:
  1570. self.writeline("context.eval_ctx.%s = " % keyword.key)
  1571. self.visit(keyword.value, frame)
  1572. try:
  1573. val = keyword.value.as_const(frame.eval_ctx)
  1574. except nodes.Impossible:
  1575. frame.eval_ctx.volatile = True
  1576. else:
  1577. setattr(frame.eval_ctx, keyword.key, val)
  1578. def visit_ScopedEvalContextModifier(self, node, frame):
  1579. old_ctx_name = self.temporary_identifier()
  1580. saved_ctx = frame.eval_ctx.save()
  1581. self.writeline("%s = context.eval_ctx.save()" % old_ctx_name)
  1582. self.visit_EvalContextModifier(node, frame)
  1583. for child in node.body:
  1584. self.visit(child, frame)
  1585. frame.eval_ctx.revert(saved_ctx)
  1586. self.writeline("context.eval_ctx.revert(%s)" % old_ctx_name)