parser.py 35 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939
  1. # -*- coding: utf-8 -*-
  2. """Parse tokens from the lexer into nodes for the compiler."""
  3. from . import nodes
  4. from ._compat import imap
  5. from .exceptions import TemplateAssertionError
  6. from .exceptions import TemplateSyntaxError
  7. from .lexer import describe_token
  8. from .lexer import describe_token_expr
  9. _statement_keywords = frozenset(
  10. [
  11. "for",
  12. "if",
  13. "block",
  14. "extends",
  15. "print",
  16. "macro",
  17. "include",
  18. "from",
  19. "import",
  20. "set",
  21. "with",
  22. "autoescape",
  23. ]
  24. )
  25. _compare_operators = frozenset(["eq", "ne", "lt", "lteq", "gt", "gteq"])
  26. _math_nodes = {
  27. "add": nodes.Add,
  28. "sub": nodes.Sub,
  29. "mul": nodes.Mul,
  30. "div": nodes.Div,
  31. "floordiv": nodes.FloorDiv,
  32. "mod": nodes.Mod,
  33. }
  34. class Parser(object):
  35. """This is the central parsing class Jinja uses. It's passed to
  36. extensions and can be used to parse expressions or statements.
  37. """
  38. def __init__(self, environment, source, name=None, filename=None, state=None):
  39. self.environment = environment
  40. self.stream = environment._tokenize(source, name, filename, state)
  41. self.name = name
  42. self.filename = filename
  43. self.closed = False
  44. self.extensions = {}
  45. for extension in environment.iter_extensions():
  46. for tag in extension.tags:
  47. self.extensions[tag] = extension.parse
  48. self._last_identifier = 0
  49. self._tag_stack = []
  50. self._end_token_stack = []
  51. def fail(self, msg, lineno=None, exc=TemplateSyntaxError):
  52. """Convenience method that raises `exc` with the message, passed
  53. line number or last line number as well as the current name and
  54. filename.
  55. """
  56. if lineno is None:
  57. lineno = self.stream.current.lineno
  58. raise exc(msg, lineno, self.name, self.filename)
  59. def _fail_ut_eof(self, name, end_token_stack, lineno):
  60. expected = []
  61. for exprs in end_token_stack:
  62. expected.extend(imap(describe_token_expr, exprs))
  63. if end_token_stack:
  64. currently_looking = " or ".join(
  65. "'%s'" % describe_token_expr(expr) for expr in end_token_stack[-1]
  66. )
  67. else:
  68. currently_looking = None
  69. if name is None:
  70. message = ["Unexpected end of template."]
  71. else:
  72. message = ["Encountered unknown tag '%s'." % name]
  73. if currently_looking:
  74. if name is not None and name in expected:
  75. message.append(
  76. "You probably made a nesting mistake. Jinja "
  77. "is expecting this tag, but currently looking "
  78. "for %s." % currently_looking
  79. )
  80. else:
  81. message.append(
  82. "Jinja was looking for the following tags: "
  83. "%s." % currently_looking
  84. )
  85. if self._tag_stack:
  86. message.append(
  87. "The innermost block that needs to be "
  88. "closed is '%s'." % self._tag_stack[-1]
  89. )
  90. self.fail(" ".join(message), lineno)
  91. def fail_unknown_tag(self, name, lineno=None):
  92. """Called if the parser encounters an unknown tag. Tries to fail
  93. with a human readable error message that could help to identify
  94. the problem.
  95. """
  96. return self._fail_ut_eof(name, self._end_token_stack, lineno)
  97. def fail_eof(self, end_tokens=None, lineno=None):
  98. """Like fail_unknown_tag but for end of template situations."""
  99. stack = list(self._end_token_stack)
  100. if end_tokens is not None:
  101. stack.append(end_tokens)
  102. return self._fail_ut_eof(None, stack, lineno)
  103. def is_tuple_end(self, extra_end_rules=None):
  104. """Are we at the end of a tuple?"""
  105. if self.stream.current.type in ("variable_end", "block_end", "rparen"):
  106. return True
  107. elif extra_end_rules is not None:
  108. return self.stream.current.test_any(extra_end_rules)
  109. return False
  110. def free_identifier(self, lineno=None):
  111. """Return a new free identifier as :class:`~jinja2.nodes.InternalName`."""
  112. self._last_identifier += 1
  113. rv = object.__new__(nodes.InternalName)
  114. nodes.Node.__init__(rv, "fi%d" % self._last_identifier, lineno=lineno)
  115. return rv
  116. def parse_statement(self):
  117. """Parse a single statement."""
  118. token = self.stream.current
  119. if token.type != "name":
  120. self.fail("tag name expected", token.lineno)
  121. self._tag_stack.append(token.value)
  122. pop_tag = True
  123. try:
  124. if token.value in _statement_keywords:
  125. return getattr(self, "parse_" + self.stream.current.value)()
  126. if token.value == "call":
  127. return self.parse_call_block()
  128. if token.value == "filter":
  129. return self.parse_filter_block()
  130. ext = self.extensions.get(token.value)
  131. if ext is not None:
  132. return ext(self)
  133. # did not work out, remove the token we pushed by accident
  134. # from the stack so that the unknown tag fail function can
  135. # produce a proper error message.
  136. self._tag_stack.pop()
  137. pop_tag = False
  138. self.fail_unknown_tag(token.value, token.lineno)
  139. finally:
  140. if pop_tag:
  141. self._tag_stack.pop()
  142. def parse_statements(self, end_tokens, drop_needle=False):
  143. """Parse multiple statements into a list until one of the end tokens
  144. is reached. This is used to parse the body of statements as it also
  145. parses template data if appropriate. The parser checks first if the
  146. current token is a colon and skips it if there is one. Then it checks
  147. for the block end and parses until if one of the `end_tokens` is
  148. reached. Per default the active token in the stream at the end of
  149. the call is the matched end token. If this is not wanted `drop_needle`
  150. can be set to `True` and the end token is removed.
  151. """
  152. # the first token may be a colon for python compatibility
  153. self.stream.skip_if("colon")
  154. # in the future it would be possible to add whole code sections
  155. # by adding some sort of end of statement token and parsing those here.
  156. self.stream.expect("block_end")
  157. result = self.subparse(end_tokens)
  158. # we reached the end of the template too early, the subparser
  159. # does not check for this, so we do that now
  160. if self.stream.current.type == "eof":
  161. self.fail_eof(end_tokens)
  162. if drop_needle:
  163. next(self.stream)
  164. return result
  165. def parse_set(self):
  166. """Parse an assign statement."""
  167. lineno = next(self.stream).lineno
  168. target = self.parse_assign_target(with_namespace=True)
  169. if self.stream.skip_if("assign"):
  170. expr = self.parse_tuple()
  171. return nodes.Assign(target, expr, lineno=lineno)
  172. filter_node = self.parse_filter(None)
  173. body = self.parse_statements(("name:endset",), drop_needle=True)
  174. return nodes.AssignBlock(target, filter_node, body, lineno=lineno)
  175. def parse_for(self):
  176. """Parse a for loop."""
  177. lineno = self.stream.expect("name:for").lineno
  178. target = self.parse_assign_target(extra_end_rules=("name:in",))
  179. self.stream.expect("name:in")
  180. iter = self.parse_tuple(
  181. with_condexpr=False, extra_end_rules=("name:recursive",)
  182. )
  183. test = None
  184. if self.stream.skip_if("name:if"):
  185. test = self.parse_expression()
  186. recursive = self.stream.skip_if("name:recursive")
  187. body = self.parse_statements(("name:endfor", "name:else"))
  188. if next(self.stream).value == "endfor":
  189. else_ = []
  190. else:
  191. else_ = self.parse_statements(("name:endfor",), drop_needle=True)
  192. return nodes.For(target, iter, body, else_, test, recursive, lineno=lineno)
  193. def parse_if(self):
  194. """Parse an if construct."""
  195. node = result = nodes.If(lineno=self.stream.expect("name:if").lineno)
  196. while 1:
  197. node.test = self.parse_tuple(with_condexpr=False)
  198. node.body = self.parse_statements(("name:elif", "name:else", "name:endif"))
  199. node.elif_ = []
  200. node.else_ = []
  201. token = next(self.stream)
  202. if token.test("name:elif"):
  203. node = nodes.If(lineno=self.stream.current.lineno)
  204. result.elif_.append(node)
  205. continue
  206. elif token.test("name:else"):
  207. result.else_ = self.parse_statements(("name:endif",), drop_needle=True)
  208. break
  209. return result
  210. def parse_with(self):
  211. node = nodes.With(lineno=next(self.stream).lineno)
  212. targets = []
  213. values = []
  214. while self.stream.current.type != "block_end":
  215. if targets:
  216. self.stream.expect("comma")
  217. target = self.parse_assign_target()
  218. target.set_ctx("param")
  219. targets.append(target)
  220. self.stream.expect("assign")
  221. values.append(self.parse_expression())
  222. node.targets = targets
  223. node.values = values
  224. node.body = self.parse_statements(("name:endwith",), drop_needle=True)
  225. return node
  226. def parse_autoescape(self):
  227. node = nodes.ScopedEvalContextModifier(lineno=next(self.stream).lineno)
  228. node.options = [nodes.Keyword("autoescape", self.parse_expression())]
  229. node.body = self.parse_statements(("name:endautoescape",), drop_needle=True)
  230. return nodes.Scope([node])
  231. def parse_block(self):
  232. node = nodes.Block(lineno=next(self.stream).lineno)
  233. node.name = self.stream.expect("name").value
  234. node.scoped = self.stream.skip_if("name:scoped")
  235. # common problem people encounter when switching from django
  236. # to jinja. we do not support hyphens in block names, so let's
  237. # raise a nicer error message in that case.
  238. if self.stream.current.type == "sub":
  239. self.fail(
  240. "Block names in Jinja have to be valid Python "
  241. "identifiers and may not contain hyphens, use an "
  242. "underscore instead."
  243. )
  244. node.body = self.parse_statements(("name:endblock",), drop_needle=True)
  245. self.stream.skip_if("name:" + node.name)
  246. return node
  247. def parse_extends(self):
  248. node = nodes.Extends(lineno=next(self.stream).lineno)
  249. node.template = self.parse_expression()
  250. return node
  251. def parse_import_context(self, node, default):
  252. if self.stream.current.test_any(
  253. "name:with", "name:without"
  254. ) and self.stream.look().test("name:context"):
  255. node.with_context = next(self.stream).value == "with"
  256. self.stream.skip()
  257. else:
  258. node.with_context = default
  259. return node
  260. def parse_include(self):
  261. node = nodes.Include(lineno=next(self.stream).lineno)
  262. node.template = self.parse_expression()
  263. if self.stream.current.test("name:ignore") and self.stream.look().test(
  264. "name:missing"
  265. ):
  266. node.ignore_missing = True
  267. self.stream.skip(2)
  268. else:
  269. node.ignore_missing = False
  270. return self.parse_import_context(node, True)
  271. def parse_import(self):
  272. node = nodes.Import(lineno=next(self.stream).lineno)
  273. node.template = self.parse_expression()
  274. self.stream.expect("name:as")
  275. node.target = self.parse_assign_target(name_only=True).name
  276. return self.parse_import_context(node, False)
  277. def parse_from(self):
  278. node = nodes.FromImport(lineno=next(self.stream).lineno)
  279. node.template = self.parse_expression()
  280. self.stream.expect("name:import")
  281. node.names = []
  282. def parse_context():
  283. if self.stream.current.value in (
  284. "with",
  285. "without",
  286. ) and self.stream.look().test("name:context"):
  287. node.with_context = next(self.stream).value == "with"
  288. self.stream.skip()
  289. return True
  290. return False
  291. while 1:
  292. if node.names:
  293. self.stream.expect("comma")
  294. if self.stream.current.type == "name":
  295. if parse_context():
  296. break
  297. target = self.parse_assign_target(name_only=True)
  298. if target.name.startswith("_"):
  299. self.fail(
  300. "names starting with an underline can not be imported",
  301. target.lineno,
  302. exc=TemplateAssertionError,
  303. )
  304. if self.stream.skip_if("name:as"):
  305. alias = self.parse_assign_target(name_only=True)
  306. node.names.append((target.name, alias.name))
  307. else:
  308. node.names.append(target.name)
  309. if parse_context() or self.stream.current.type != "comma":
  310. break
  311. else:
  312. self.stream.expect("name")
  313. if not hasattr(node, "with_context"):
  314. node.with_context = False
  315. return node
  316. def parse_signature(self, node):
  317. node.args = args = []
  318. node.defaults = defaults = []
  319. self.stream.expect("lparen")
  320. while self.stream.current.type != "rparen":
  321. if args:
  322. self.stream.expect("comma")
  323. arg = self.parse_assign_target(name_only=True)
  324. arg.set_ctx("param")
  325. if self.stream.skip_if("assign"):
  326. defaults.append(self.parse_expression())
  327. elif defaults:
  328. self.fail("non-default argument follows default argument")
  329. args.append(arg)
  330. self.stream.expect("rparen")
  331. def parse_call_block(self):
  332. node = nodes.CallBlock(lineno=next(self.stream).lineno)
  333. if self.stream.current.type == "lparen":
  334. self.parse_signature(node)
  335. else:
  336. node.args = []
  337. node.defaults = []
  338. node.call = self.parse_expression()
  339. if not isinstance(node.call, nodes.Call):
  340. self.fail("expected call", node.lineno)
  341. node.body = self.parse_statements(("name:endcall",), drop_needle=True)
  342. return node
  343. def parse_filter_block(self):
  344. node = nodes.FilterBlock(lineno=next(self.stream).lineno)
  345. node.filter = self.parse_filter(None, start_inline=True)
  346. node.body = self.parse_statements(("name:endfilter",), drop_needle=True)
  347. return node
  348. def parse_macro(self):
  349. node = nodes.Macro(lineno=next(self.stream).lineno)
  350. node.name = self.parse_assign_target(name_only=True).name
  351. self.parse_signature(node)
  352. node.body = self.parse_statements(("name:endmacro",), drop_needle=True)
  353. return node
  354. def parse_print(self):
  355. node = nodes.Output(lineno=next(self.stream).lineno)
  356. node.nodes = []
  357. while self.stream.current.type != "block_end":
  358. if node.nodes:
  359. self.stream.expect("comma")
  360. node.nodes.append(self.parse_expression())
  361. return node
  362. def parse_assign_target(
  363. self,
  364. with_tuple=True,
  365. name_only=False,
  366. extra_end_rules=None,
  367. with_namespace=False,
  368. ):
  369. """Parse an assignment target. As Jinja allows assignments to
  370. tuples, this function can parse all allowed assignment targets. Per
  371. default assignments to tuples are parsed, that can be disable however
  372. by setting `with_tuple` to `False`. If only assignments to names are
  373. wanted `name_only` can be set to `True`. The `extra_end_rules`
  374. parameter is forwarded to the tuple parsing function. If
  375. `with_namespace` is enabled, a namespace assignment may be parsed.
  376. """
  377. if with_namespace and self.stream.look().type == "dot":
  378. token = self.stream.expect("name")
  379. next(self.stream) # dot
  380. attr = self.stream.expect("name")
  381. target = nodes.NSRef(token.value, attr.value, lineno=token.lineno)
  382. elif name_only:
  383. token = self.stream.expect("name")
  384. target = nodes.Name(token.value, "store", lineno=token.lineno)
  385. else:
  386. if with_tuple:
  387. target = self.parse_tuple(
  388. simplified=True, extra_end_rules=extra_end_rules
  389. )
  390. else:
  391. target = self.parse_primary()
  392. target.set_ctx("store")
  393. if not target.can_assign():
  394. self.fail(
  395. "can't assign to %r" % target.__class__.__name__.lower(), target.lineno
  396. )
  397. return target
  398. def parse_expression(self, with_condexpr=True):
  399. """Parse an expression. Per default all expressions are parsed, if
  400. the optional `with_condexpr` parameter is set to `False` conditional
  401. expressions are not parsed.
  402. """
  403. if with_condexpr:
  404. return self.parse_condexpr()
  405. return self.parse_or()
  406. def parse_condexpr(self):
  407. lineno = self.stream.current.lineno
  408. expr1 = self.parse_or()
  409. while self.stream.skip_if("name:if"):
  410. expr2 = self.parse_or()
  411. if self.stream.skip_if("name:else"):
  412. expr3 = self.parse_condexpr()
  413. else:
  414. expr3 = None
  415. expr1 = nodes.CondExpr(expr2, expr1, expr3, lineno=lineno)
  416. lineno = self.stream.current.lineno
  417. return expr1
  418. def parse_or(self):
  419. lineno = self.stream.current.lineno
  420. left = self.parse_and()
  421. while self.stream.skip_if("name:or"):
  422. right = self.parse_and()
  423. left = nodes.Or(left, right, lineno=lineno)
  424. lineno = self.stream.current.lineno
  425. return left
  426. def parse_and(self):
  427. lineno = self.stream.current.lineno
  428. left = self.parse_not()
  429. while self.stream.skip_if("name:and"):
  430. right = self.parse_not()
  431. left = nodes.And(left, right, lineno=lineno)
  432. lineno = self.stream.current.lineno
  433. return left
  434. def parse_not(self):
  435. if self.stream.current.test("name:not"):
  436. lineno = next(self.stream).lineno
  437. return nodes.Not(self.parse_not(), lineno=lineno)
  438. return self.parse_compare()
  439. def parse_compare(self):
  440. lineno = self.stream.current.lineno
  441. expr = self.parse_math1()
  442. ops = []
  443. while 1:
  444. token_type = self.stream.current.type
  445. if token_type in _compare_operators:
  446. next(self.stream)
  447. ops.append(nodes.Operand(token_type, self.parse_math1()))
  448. elif self.stream.skip_if("name:in"):
  449. ops.append(nodes.Operand("in", self.parse_math1()))
  450. elif self.stream.current.test("name:not") and self.stream.look().test(
  451. "name:in"
  452. ):
  453. self.stream.skip(2)
  454. ops.append(nodes.Operand("notin", self.parse_math1()))
  455. else:
  456. break
  457. lineno = self.stream.current.lineno
  458. if not ops:
  459. return expr
  460. return nodes.Compare(expr, ops, lineno=lineno)
  461. def parse_math1(self):
  462. lineno = self.stream.current.lineno
  463. left = self.parse_concat()
  464. while self.stream.current.type in ("add", "sub"):
  465. cls = _math_nodes[self.stream.current.type]
  466. next(self.stream)
  467. right = self.parse_concat()
  468. left = cls(left, right, lineno=lineno)
  469. lineno = self.stream.current.lineno
  470. return left
  471. def parse_concat(self):
  472. lineno = self.stream.current.lineno
  473. args = [self.parse_math2()]
  474. while self.stream.current.type == "tilde":
  475. next(self.stream)
  476. args.append(self.parse_math2())
  477. if len(args) == 1:
  478. return args[0]
  479. return nodes.Concat(args, lineno=lineno)
  480. def parse_math2(self):
  481. lineno = self.stream.current.lineno
  482. left = self.parse_pow()
  483. while self.stream.current.type in ("mul", "div", "floordiv", "mod"):
  484. cls = _math_nodes[self.stream.current.type]
  485. next(self.stream)
  486. right = self.parse_pow()
  487. left = cls(left, right, lineno=lineno)
  488. lineno = self.stream.current.lineno
  489. return left
  490. def parse_pow(self):
  491. lineno = self.stream.current.lineno
  492. left = self.parse_unary()
  493. while self.stream.current.type == "pow":
  494. next(self.stream)
  495. right = self.parse_unary()
  496. left = nodes.Pow(left, right, lineno=lineno)
  497. lineno = self.stream.current.lineno
  498. return left
  499. def parse_unary(self, with_filter=True):
  500. token_type = self.stream.current.type
  501. lineno = self.stream.current.lineno
  502. if token_type == "sub":
  503. next(self.stream)
  504. node = nodes.Neg(self.parse_unary(False), lineno=lineno)
  505. elif token_type == "add":
  506. next(self.stream)
  507. node = nodes.Pos(self.parse_unary(False), lineno=lineno)
  508. else:
  509. node = self.parse_primary()
  510. node = self.parse_postfix(node)
  511. if with_filter:
  512. node = self.parse_filter_expr(node)
  513. return node
  514. def parse_primary(self):
  515. token = self.stream.current
  516. if token.type == "name":
  517. if token.value in ("true", "false", "True", "False"):
  518. node = nodes.Const(token.value in ("true", "True"), lineno=token.lineno)
  519. elif token.value in ("none", "None"):
  520. node = nodes.Const(None, lineno=token.lineno)
  521. else:
  522. node = nodes.Name(token.value, "load", lineno=token.lineno)
  523. next(self.stream)
  524. elif token.type == "string":
  525. next(self.stream)
  526. buf = [token.value]
  527. lineno = token.lineno
  528. while self.stream.current.type == "string":
  529. buf.append(self.stream.current.value)
  530. next(self.stream)
  531. node = nodes.Const("".join(buf), lineno=lineno)
  532. elif token.type in ("integer", "float"):
  533. next(self.stream)
  534. node = nodes.Const(token.value, lineno=token.lineno)
  535. elif token.type == "lparen":
  536. next(self.stream)
  537. node = self.parse_tuple(explicit_parentheses=True)
  538. self.stream.expect("rparen")
  539. elif token.type == "lbracket":
  540. node = self.parse_list()
  541. elif token.type == "lbrace":
  542. node = self.parse_dict()
  543. else:
  544. self.fail("unexpected '%s'" % describe_token(token), token.lineno)
  545. return node
  546. def parse_tuple(
  547. self,
  548. simplified=False,
  549. with_condexpr=True,
  550. extra_end_rules=None,
  551. explicit_parentheses=False,
  552. ):
  553. """Works like `parse_expression` but if multiple expressions are
  554. delimited by a comma a :class:`~jinja2.nodes.Tuple` node is created.
  555. This method could also return a regular expression instead of a tuple
  556. if no commas where found.
  557. The default parsing mode is a full tuple. If `simplified` is `True`
  558. only names and literals are parsed. The `no_condexpr` parameter is
  559. forwarded to :meth:`parse_expression`.
  560. Because tuples do not require delimiters and may end in a bogus comma
  561. an extra hint is needed that marks the end of a tuple. For example
  562. for loops support tuples between `for` and `in`. In that case the
  563. `extra_end_rules` is set to ``['name:in']``.
  564. `explicit_parentheses` is true if the parsing was triggered by an
  565. expression in parentheses. This is used to figure out if an empty
  566. tuple is a valid expression or not.
  567. """
  568. lineno = self.stream.current.lineno
  569. if simplified:
  570. parse = self.parse_primary
  571. elif with_condexpr:
  572. parse = self.parse_expression
  573. else:
  574. def parse():
  575. return self.parse_expression(with_condexpr=False)
  576. args = []
  577. is_tuple = False
  578. while 1:
  579. if args:
  580. self.stream.expect("comma")
  581. if self.is_tuple_end(extra_end_rules):
  582. break
  583. args.append(parse())
  584. if self.stream.current.type == "comma":
  585. is_tuple = True
  586. else:
  587. break
  588. lineno = self.stream.current.lineno
  589. if not is_tuple:
  590. if args:
  591. return args[0]
  592. # if we don't have explicit parentheses, an empty tuple is
  593. # not a valid expression. This would mean nothing (literally
  594. # nothing) in the spot of an expression would be an empty
  595. # tuple.
  596. if not explicit_parentheses:
  597. self.fail(
  598. "Expected an expression, got '%s'"
  599. % describe_token(self.stream.current)
  600. )
  601. return nodes.Tuple(args, "load", lineno=lineno)
  602. def parse_list(self):
  603. token = self.stream.expect("lbracket")
  604. items = []
  605. while self.stream.current.type != "rbracket":
  606. if items:
  607. self.stream.expect("comma")
  608. if self.stream.current.type == "rbracket":
  609. break
  610. items.append(self.parse_expression())
  611. self.stream.expect("rbracket")
  612. return nodes.List(items, lineno=token.lineno)
  613. def parse_dict(self):
  614. token = self.stream.expect("lbrace")
  615. items = []
  616. while self.stream.current.type != "rbrace":
  617. if items:
  618. self.stream.expect("comma")
  619. if self.stream.current.type == "rbrace":
  620. break
  621. key = self.parse_expression()
  622. self.stream.expect("colon")
  623. value = self.parse_expression()
  624. items.append(nodes.Pair(key, value, lineno=key.lineno))
  625. self.stream.expect("rbrace")
  626. return nodes.Dict(items, lineno=token.lineno)
  627. def parse_postfix(self, node):
  628. while 1:
  629. token_type = self.stream.current.type
  630. if token_type == "dot" or token_type == "lbracket":
  631. node = self.parse_subscript(node)
  632. # calls are valid both after postfix expressions (getattr
  633. # and getitem) as well as filters and tests
  634. elif token_type == "lparen":
  635. node = self.parse_call(node)
  636. else:
  637. break
  638. return node
  639. def parse_filter_expr(self, node):
  640. while 1:
  641. token_type = self.stream.current.type
  642. if token_type == "pipe":
  643. node = self.parse_filter(node)
  644. elif token_type == "name" and self.stream.current.value == "is":
  645. node = self.parse_test(node)
  646. # calls are valid both after postfix expressions (getattr
  647. # and getitem) as well as filters and tests
  648. elif token_type == "lparen":
  649. node = self.parse_call(node)
  650. else:
  651. break
  652. return node
  653. def parse_subscript(self, node):
  654. token = next(self.stream)
  655. if token.type == "dot":
  656. attr_token = self.stream.current
  657. next(self.stream)
  658. if attr_token.type == "name":
  659. return nodes.Getattr(
  660. node, attr_token.value, "load", lineno=token.lineno
  661. )
  662. elif attr_token.type != "integer":
  663. self.fail("expected name or number", attr_token.lineno)
  664. arg = nodes.Const(attr_token.value, lineno=attr_token.lineno)
  665. return nodes.Getitem(node, arg, "load", lineno=token.lineno)
  666. if token.type == "lbracket":
  667. args = []
  668. while self.stream.current.type != "rbracket":
  669. if args:
  670. self.stream.expect("comma")
  671. args.append(self.parse_subscribed())
  672. self.stream.expect("rbracket")
  673. if len(args) == 1:
  674. arg = args[0]
  675. else:
  676. arg = nodes.Tuple(args, "load", lineno=token.lineno)
  677. return nodes.Getitem(node, arg, "load", lineno=token.lineno)
  678. self.fail("expected subscript expression", token.lineno)
  679. def parse_subscribed(self):
  680. lineno = self.stream.current.lineno
  681. if self.stream.current.type == "colon":
  682. next(self.stream)
  683. args = [None]
  684. else:
  685. node = self.parse_expression()
  686. if self.stream.current.type != "colon":
  687. return node
  688. next(self.stream)
  689. args = [node]
  690. if self.stream.current.type == "colon":
  691. args.append(None)
  692. elif self.stream.current.type not in ("rbracket", "comma"):
  693. args.append(self.parse_expression())
  694. else:
  695. args.append(None)
  696. if self.stream.current.type == "colon":
  697. next(self.stream)
  698. if self.stream.current.type not in ("rbracket", "comma"):
  699. args.append(self.parse_expression())
  700. else:
  701. args.append(None)
  702. else:
  703. args.append(None)
  704. return nodes.Slice(lineno=lineno, *args)
  705. def parse_call(self, node):
  706. token = self.stream.expect("lparen")
  707. args = []
  708. kwargs = []
  709. dyn_args = dyn_kwargs = None
  710. require_comma = False
  711. def ensure(expr):
  712. if not expr:
  713. self.fail("invalid syntax for function call expression", token.lineno)
  714. while self.stream.current.type != "rparen":
  715. if require_comma:
  716. self.stream.expect("comma")
  717. # support for trailing comma
  718. if self.stream.current.type == "rparen":
  719. break
  720. if self.stream.current.type == "mul":
  721. ensure(dyn_args is None and dyn_kwargs is None)
  722. next(self.stream)
  723. dyn_args = self.parse_expression()
  724. elif self.stream.current.type == "pow":
  725. ensure(dyn_kwargs is None)
  726. next(self.stream)
  727. dyn_kwargs = self.parse_expression()
  728. else:
  729. if (
  730. self.stream.current.type == "name"
  731. and self.stream.look().type == "assign"
  732. ):
  733. # Parsing a kwarg
  734. ensure(dyn_kwargs is None)
  735. key = self.stream.current.value
  736. self.stream.skip(2)
  737. value = self.parse_expression()
  738. kwargs.append(nodes.Keyword(key, value, lineno=value.lineno))
  739. else:
  740. # Parsing an arg
  741. ensure(dyn_args is None and dyn_kwargs is None and not kwargs)
  742. args.append(self.parse_expression())
  743. require_comma = True
  744. self.stream.expect("rparen")
  745. if node is None:
  746. return args, kwargs, dyn_args, dyn_kwargs
  747. return nodes.Call(node, args, kwargs, dyn_args, dyn_kwargs, lineno=token.lineno)
  748. def parse_filter(self, node, start_inline=False):
  749. while self.stream.current.type == "pipe" or start_inline:
  750. if not start_inline:
  751. next(self.stream)
  752. token = self.stream.expect("name")
  753. name = token.value
  754. while self.stream.current.type == "dot":
  755. next(self.stream)
  756. name += "." + self.stream.expect("name").value
  757. if self.stream.current.type == "lparen":
  758. args, kwargs, dyn_args, dyn_kwargs = self.parse_call(None)
  759. else:
  760. args = []
  761. kwargs = []
  762. dyn_args = dyn_kwargs = None
  763. node = nodes.Filter(
  764. node, name, args, kwargs, dyn_args, dyn_kwargs, lineno=token.lineno
  765. )
  766. start_inline = False
  767. return node
  768. def parse_test(self, node):
  769. token = next(self.stream)
  770. if self.stream.current.test("name:not"):
  771. next(self.stream)
  772. negated = True
  773. else:
  774. negated = False
  775. name = self.stream.expect("name").value
  776. while self.stream.current.type == "dot":
  777. next(self.stream)
  778. name += "." + self.stream.expect("name").value
  779. dyn_args = dyn_kwargs = None
  780. kwargs = []
  781. if self.stream.current.type == "lparen":
  782. args, kwargs, dyn_args, dyn_kwargs = self.parse_call(None)
  783. elif self.stream.current.type in (
  784. "name",
  785. "string",
  786. "integer",
  787. "float",
  788. "lparen",
  789. "lbracket",
  790. "lbrace",
  791. ) and not self.stream.current.test_any("name:else", "name:or", "name:and"):
  792. if self.stream.current.test("name:is"):
  793. self.fail("You cannot chain multiple tests with is")
  794. arg_node = self.parse_primary()
  795. arg_node = self.parse_postfix(arg_node)
  796. args = [arg_node]
  797. else:
  798. args = []
  799. node = nodes.Test(
  800. node, name, args, kwargs, dyn_args, dyn_kwargs, lineno=token.lineno
  801. )
  802. if negated:
  803. node = nodes.Not(node, lineno=token.lineno)
  804. return node
  805. def subparse(self, end_tokens=None):
  806. body = []
  807. data_buffer = []
  808. add_data = data_buffer.append
  809. if end_tokens is not None:
  810. self._end_token_stack.append(end_tokens)
  811. def flush_data():
  812. if data_buffer:
  813. lineno = data_buffer[0].lineno
  814. body.append(nodes.Output(data_buffer[:], lineno=lineno))
  815. del data_buffer[:]
  816. try:
  817. while self.stream:
  818. token = self.stream.current
  819. if token.type == "data":
  820. if token.value:
  821. add_data(nodes.TemplateData(token.value, lineno=token.lineno))
  822. next(self.stream)
  823. elif token.type == "variable_begin":
  824. next(self.stream)
  825. add_data(self.parse_tuple(with_condexpr=True))
  826. self.stream.expect("variable_end")
  827. elif token.type == "block_begin":
  828. flush_data()
  829. next(self.stream)
  830. if end_tokens is not None and self.stream.current.test_any(
  831. *end_tokens
  832. ):
  833. return body
  834. rv = self.parse_statement()
  835. if isinstance(rv, list):
  836. body.extend(rv)
  837. else:
  838. body.append(rv)
  839. self.stream.expect("block_end")
  840. else:
  841. raise AssertionError("internal parsing error")
  842. flush_data()
  843. finally:
  844. if end_tokens is not None:
  845. self._end_token_stack.pop()
  846. return body
  847. def parse(self):
  848. """Parse the whole template into a `Template` node."""
  849. result = nodes.Template(self.subparse(), lineno=1)
  850. result.set_environment(self.environment)
  851. return result