jinja2.parser

   1"""Parse tokens from the lexer into nodes for the compiler."""
   2import typing
   3import typing as t
   4
   5from . import nodes
   6from .exceptions import TemplateAssertionError
   7from .exceptions import TemplateSyntaxError
   8from .lexer import describe_token
   9from .lexer import describe_token_expr
  10
  11if t.TYPE_CHECKING:
  12    import typing_extensions as te
  13    from .environment import Environment
  14
  15_ImportInclude = t.TypeVar("_ImportInclude", nodes.Import, nodes.Include)
  16_MacroCall = t.TypeVar("_MacroCall", nodes.Macro, nodes.CallBlock)
  17
  18_statement_keywords = frozenset(
  19    [
  20        "for",
  21        "if",
  22        "block",
  23        "extends",
  24        "print",
  25        "macro",
  26        "include",
  27        "from",
  28        "import",
  29        "set",
  30        "with",
  31        "autoescape",
  32    ]
  33)
  34_compare_operators = frozenset(["eq", "ne", "lt", "lteq", "gt", "gteq"])
  35
  36_math_nodes: t.Dict[str, t.Type[nodes.Expr]] = {
  37    "add": nodes.Add,
  38    "sub": nodes.Sub,
  39    "mul": nodes.Mul,
  40    "div": nodes.Div,
  41    "floordiv": nodes.FloorDiv,
  42    "mod": nodes.Mod,
  43}
  44
  45
  46class Parser:
  47    """This is the central parsing class Jinja uses.  It's passed to
  48    extensions and can be used to parse expressions or statements.
  49    """
  50
  51    def __init__(
  52        self,
  53        environment: "Environment",
  54        source: str,
  55        name: t.Optional[str] = None,
  56        filename: t.Optional[str] = None,
  57        state: t.Optional[str] = None,
  58    ) -> None:
  59        self.environment = environment
  60        self.stream = environment._tokenize(source, name, filename, state)
  61        self.name = name
  62        self.filename = filename
  63        self.closed = False
  64        self.extensions: t.Dict[
  65            str, t.Callable[["Parser"], t.Union[nodes.Node, t.List[nodes.Node]]]
  66        ] = {}
  67        for extension in environment.iter_extensions():
  68            for tag in extension.tags:
  69                self.extensions[tag] = extension.parse
  70        self._last_identifier = 0
  71        self._tag_stack: t.List[str] = []
  72        self._end_token_stack: t.List[t.Tuple[str, ...]] = []
  73
  74    def fail(
  75        self,
  76        msg: str,
  77        lineno: t.Optional[int] = None,
  78        exc: t.Type[TemplateSyntaxError] = TemplateSyntaxError,
  79    ) -> "te.NoReturn":
  80        """Convenience method that raises `exc` with the message, passed
  81        line number or last line number as well as the current name and
  82        filename.
  83        """
  84        if lineno is None:
  85            lineno = self.stream.current.lineno
  86        raise exc(msg, lineno, self.name, self.filename)
  87
  88    def _fail_ut_eof(
  89        self,
  90        name: t.Optional[str],
  91        end_token_stack: t.List[t.Tuple[str, ...]],
  92        lineno: t.Optional[int],
  93    ) -> "te.NoReturn":
  94        expected: t.Set[str] = set()
  95        for exprs in end_token_stack:
  96            expected.update(map(describe_token_expr, exprs))
  97        if end_token_stack:
  98            currently_looking: t.Optional[str] = " or ".join(
  99                map(repr, map(describe_token_expr, end_token_stack[-1]))
 100            )
 101        else:
 102            currently_looking = None
 103
 104        if name is None:
 105            message = ["Unexpected end of template."]
 106        else:
 107            message = [f"Encountered unknown tag {name!r}."]
 108
 109        if currently_looking:
 110            if name is not None and name in expected:
 111                message.append(
 112                    "You probably made a nesting mistake. Jinja is expecting this tag,"
 113                    f" but currently looking for {currently_looking}."
 114                )
 115            else:
 116                message.append(
 117                    f"Jinja was looking for the following tags: {currently_looking}."
 118                )
 119
 120        if self._tag_stack:
 121            message.append(
 122                "The innermost block that needs to be closed is"
 123                f" {self._tag_stack[-1]!r}."
 124            )
 125
 126        self.fail(" ".join(message), lineno)
 127
 128    def fail_unknown_tag(
 129        self, name: str, lineno: t.Optional[int] = None
 130    ) -> "te.NoReturn":
 131        """Called if the parser encounters an unknown tag.  Tries to fail
 132        with a human readable error message that could help to identify
 133        the problem.
 134        """
 135        self._fail_ut_eof(name, self._end_token_stack, lineno)
 136
 137    def fail_eof(
 138        self,
 139        end_tokens: t.Optional[t.Tuple[str, ...]] = None,
 140        lineno: t.Optional[int] = None,
 141    ) -> "te.NoReturn":
 142        """Like fail_unknown_tag but for end of template situations."""
 143        stack = list(self._end_token_stack)
 144        if end_tokens is not None:
 145            stack.append(end_tokens)
 146        self._fail_ut_eof(None, stack, lineno)
 147
 148    def is_tuple_end(
 149        self, extra_end_rules: t.Optional[t.Tuple[str, ...]] = None
 150    ) -> bool:
 151        """Are we at the end of a tuple?"""
 152        if self.stream.current.type in ("variable_end", "block_end", "rparen"):
 153            return True
 154        elif extra_end_rules is not None:
 155            return self.stream.current.test_any(extra_end_rules)  # type: ignore
 156        return False
 157
 158    def free_identifier(self, lineno: t.Optional[int] = None) -> nodes.InternalName:
 159        """Return a new free identifier as :class:`~jinja2.nodes.InternalName`."""
 160        self._last_identifier += 1
 161        rv = object.__new__(nodes.InternalName)
 162        nodes.Node.__init__(rv, f"fi{self._last_identifier}", lineno=lineno)
 163        return rv
 164
 165    def parse_statement(self) -> t.Union[nodes.Node, t.List[nodes.Node]]:
 166        """Parse a single statement."""
 167        token = self.stream.current
 168        if token.type != "name":
 169            self.fail("tag name expected", token.lineno)
 170        self._tag_stack.append(token.value)
 171        pop_tag = True
 172        try:
 173            if token.value in _statement_keywords:
 174                f = getattr(self, f"parse_{self.stream.current.value}")
 175                return f()  # type: ignore
 176            if token.value == "call":
 177                return self.parse_call_block()
 178            if token.value == "filter":
 179                return self.parse_filter_block()
 180            ext = self.extensions.get(token.value)
 181            if ext is not None:
 182                return ext(self)
 183
 184            # did not work out, remove the token we pushed by accident
 185            # from the stack so that the unknown tag fail function can
 186            # produce a proper error message.
 187            self._tag_stack.pop()
 188            pop_tag = False
 189            self.fail_unknown_tag(token.value, token.lineno)
 190        finally:
 191            if pop_tag:
 192                self._tag_stack.pop()
 193
 194    def parse_statements(
 195        self, end_tokens: t.Tuple[str, ...], drop_needle: bool = False
 196    ) -> t.List[nodes.Node]:
 197        """Parse multiple statements into a list until one of the end tokens
 198        is reached.  This is used to parse the body of statements as it also
 199        parses template data if appropriate.  The parser checks first if the
 200        current token is a colon and skips it if there is one.  Then it checks
 201        for the block end and parses until if one of the `end_tokens` is
 202        reached.  Per default the active token in the stream at the end of
 203        the call is the matched end token.  If this is not wanted `drop_needle`
 204        can be set to `True` and the end token is removed.
 205        """
 206        # the first token may be a colon for python compatibility
 207        self.stream.skip_if("colon")
 208
 209        # in the future it would be possible to add whole code sections
 210        # by adding some sort of end of statement token and parsing those here.
 211        self.stream.expect("block_end")
 212        result = self.subparse(end_tokens)
 213
 214        # we reached the end of the template too early, the subparser
 215        # does not check for this, so we do that now
 216        if self.stream.current.type == "eof":
 217            self.fail_eof(end_tokens)
 218
 219        if drop_needle:
 220            next(self.stream)
 221        return result
 222
 223    def parse_set(self) -> t.Union[nodes.Assign, nodes.AssignBlock]:
 224        """Parse an assign statement."""
 225        lineno = next(self.stream).lineno
 226        target = self.parse_assign_target(with_namespace=True)
 227        if self.stream.skip_if("assign"):
 228            expr = self.parse_tuple()
 229            return nodes.Assign(target, expr, lineno=lineno)
 230        filter_node = self.parse_filter(None)
 231        body = self.parse_statements(("name:endset",), drop_needle=True)
 232        return nodes.AssignBlock(target, filter_node, body, lineno=lineno)
 233
 234    def parse_for(self) -> nodes.For:
 235        """Parse a for loop."""
 236        lineno = self.stream.expect("name:for").lineno
 237        target = self.parse_assign_target(extra_end_rules=("name:in",))
 238        self.stream.expect("name:in")
 239        iter = self.parse_tuple(
 240            with_condexpr=False, extra_end_rules=("name:recursive",)
 241        )
 242        test = None
 243        if self.stream.skip_if("name:if"):
 244            test = self.parse_expression()
 245        recursive = self.stream.skip_if("name:recursive")
 246        body = self.parse_statements(("name:endfor", "name:else"))
 247        if next(self.stream).value == "endfor":
 248            else_ = []
 249        else:
 250            else_ = self.parse_statements(("name:endfor",), drop_needle=True)
 251        return nodes.For(target, iter, body, else_, test, recursive, lineno=lineno)
 252
 253    def parse_if(self) -> nodes.If:
 254        """Parse an if construct."""
 255        node = result = nodes.If(lineno=self.stream.expect("name:if").lineno)
 256        while True:
 257            node.test = self.parse_tuple(with_condexpr=False)
 258            node.body = self.parse_statements(("name:elif", "name:else", "name:endif"))
 259            node.elif_ = []
 260            node.else_ = []
 261            token = next(self.stream)
 262            if token.test("name:elif"):
 263                node = nodes.If(lineno=self.stream.current.lineno)
 264                result.elif_.append(node)
 265                continue
 266            elif token.test("name:else"):
 267                result.else_ = self.parse_statements(("name:endif",), drop_needle=True)
 268            break
 269        return result
 270
 271    def parse_with(self) -> nodes.With:
 272        node = nodes.With(lineno=next(self.stream).lineno)
 273        targets: t.List[nodes.Expr] = []
 274        values: t.List[nodes.Expr] = []
 275        while self.stream.current.type != "block_end":
 276            if targets:
 277                self.stream.expect("comma")
 278            target = self.parse_assign_target()
 279            target.set_ctx("param")
 280            targets.append(target)
 281            self.stream.expect("assign")
 282            values.append(self.parse_expression())
 283        node.targets = targets
 284        node.values = values
 285        node.body = self.parse_statements(("name:endwith",), drop_needle=True)
 286        return node
 287
 288    def parse_autoescape(self) -> nodes.Scope:
 289        node = nodes.ScopedEvalContextModifier(lineno=next(self.stream).lineno)
 290        node.options = [nodes.Keyword("autoescape", self.parse_expression())]
 291        node.body = self.parse_statements(("name:endautoescape",), drop_needle=True)
 292        return nodes.Scope([node])
 293
 294    def parse_block(self) -> nodes.Block:
 295        node = nodes.Block(lineno=next(self.stream).lineno)
 296        node.name = self.stream.expect("name").value
 297        node.scoped = self.stream.skip_if("name:scoped")
 298        node.required = self.stream.skip_if("name:required")
 299
 300        # common problem people encounter when switching from django
 301        # to jinja.  we do not support hyphens in block names, so let's
 302        # raise a nicer error message in that case.
 303        if self.stream.current.type == "sub":
 304            self.fail(
 305                "Block names in Jinja have to be valid Python identifiers and may not"
 306                " contain hyphens, use an underscore instead."
 307            )
 308
 309        node.body = self.parse_statements(("name:endblock",), drop_needle=True)
 310
 311        # enforce that required blocks only contain whitespace or comments
 312        # by asserting that the body, if not empty, is just TemplateData nodes
 313        # with whitespace data
 314        if node.required:
 315            for body_node in node.body:
 316                if not isinstance(body_node, nodes.Output) or any(
 317                    not isinstance(output_node, nodes.TemplateData)
 318                    or not output_node.data.isspace()
 319                    for output_node in body_node.nodes
 320                ):
 321                    self.fail("Required blocks can only contain comments or whitespace")
 322
 323        self.stream.skip_if("name:" + node.name)
 324        return node
 325
 326    def parse_extends(self) -> nodes.Extends:
 327        node = nodes.Extends(lineno=next(self.stream).lineno)
 328        node.template = self.parse_expression()
 329        return node
 330
 331    def parse_import_context(
 332        self, node: _ImportInclude, default: bool
 333    ) -> _ImportInclude:
 334        if self.stream.current.test_any(
 335            "name:with", "name:without"
 336        ) and self.stream.look().test("name:context"):
 337            node.with_context = next(self.stream).value == "with"
 338            self.stream.skip()
 339        else:
 340            node.with_context = default
 341        return node
 342
 343    def parse_include(self) -> nodes.Include:
 344        node = nodes.Include(lineno=next(self.stream).lineno)
 345        node.template = self.parse_expression()
 346        if self.stream.current.test("name:ignore") and self.stream.look().test(
 347            "name:missing"
 348        ):
 349            node.ignore_missing = True
 350            self.stream.skip(2)
 351        else:
 352            node.ignore_missing = False
 353        return self.parse_import_context(node, True)
 354
 355    def parse_import(self) -> nodes.Import:
 356        node = nodes.Import(lineno=next(self.stream).lineno)
 357        node.template = self.parse_expression()
 358        self.stream.expect("name:as")
 359        node.target = self.parse_assign_target(name_only=True).name
 360        return self.parse_import_context(node, False)
 361
 362    def parse_from(self) -> nodes.FromImport:
 363        node = nodes.FromImport(lineno=next(self.stream).lineno)
 364        node.template = self.parse_expression()
 365        self.stream.expect("name:import")
 366        node.names = []
 367
 368        def parse_context() -> bool:
 369            if self.stream.current.value in {
 370                "with",
 371                "without",
 372            } and self.stream.look().test("name:context"):
 373                node.with_context = next(self.stream).value == "with"
 374                self.stream.skip()
 375                return True
 376            return False
 377
 378        while True:
 379            if node.names:
 380                self.stream.expect("comma")
 381            if self.stream.current.type == "name":
 382                if parse_context():
 383                    break
 384                target = self.parse_assign_target(name_only=True)
 385                if target.name.startswith("_"):
 386                    self.fail(
 387                        "names starting with an underline can not be imported",
 388                        target.lineno,
 389                        exc=TemplateAssertionError,
 390                    )
 391                if self.stream.skip_if("name:as"):
 392                    alias = self.parse_assign_target(name_only=True)
 393                    node.names.append((target.name, alias.name))
 394                else:
 395                    node.names.append(target.name)
 396                if parse_context() or self.stream.current.type != "comma":
 397                    break
 398            else:
 399                self.stream.expect("name")
 400        if not hasattr(node, "with_context"):
 401            node.with_context = False
 402        return node
 403
 404    def parse_signature(self, node: _MacroCall) -> None:
 405        args = node.args = []
 406        defaults = node.defaults = []
 407        self.stream.expect("lparen")
 408        while self.stream.current.type != "rparen":
 409            if args:
 410                self.stream.expect("comma")
 411            arg = self.parse_assign_target(name_only=True)
 412            arg.set_ctx("param")
 413            if self.stream.skip_if("assign"):
 414                defaults.append(self.parse_expression())
 415            elif defaults:
 416                self.fail("non-default argument follows default argument")
 417            args.append(arg)
 418        self.stream.expect("rparen")
 419
 420    def parse_call_block(self) -> nodes.CallBlock:
 421        node = nodes.CallBlock(lineno=next(self.stream).lineno)
 422        if self.stream.current.type == "lparen":
 423            self.parse_signature(node)
 424        else:
 425            node.args = []
 426            node.defaults = []
 427
 428        call_node = self.parse_expression()
 429        if not isinstance(call_node, nodes.Call):
 430            self.fail("expected call", node.lineno)
 431        node.call = call_node
 432        node.body = self.parse_statements(("name:endcall",), drop_needle=True)
 433        return node
 434
 435    def parse_filter_block(self) -> nodes.FilterBlock:
 436        node = nodes.FilterBlock(lineno=next(self.stream).lineno)
 437        node.filter = self.parse_filter(None, start_inline=True)  # type: ignore
 438        node.body = self.parse_statements(("name:endfilter",), drop_needle=True)
 439        return node
 440
 441    def parse_macro(self) -> nodes.Macro:
 442        node = nodes.Macro(lineno=next(self.stream).lineno)
 443        node.name = self.parse_assign_target(name_only=True).name
 444        self.parse_signature(node)
 445        node.body = self.parse_statements(("name:endmacro",), drop_needle=True)
 446        return node
 447
 448    def parse_print(self) -> nodes.Output:
 449        node = nodes.Output(lineno=next(self.stream).lineno)
 450        node.nodes = []
 451        while self.stream.current.type != "block_end":
 452            if node.nodes:
 453                self.stream.expect("comma")
 454            node.nodes.append(self.parse_expression())
 455        return node
 456
 457    @typing.overload
 458    def parse_assign_target(
 459        self, with_tuple: bool = ..., name_only: "te.Literal[True]" = ...
 460    ) -> nodes.Name:
 461        ...
 462
 463    @typing.overload
 464    def parse_assign_target(
 465        self,
 466        with_tuple: bool = True,
 467        name_only: bool = False,
 468        extra_end_rules: t.Optional[t.Tuple[str, ...]] = None,
 469        with_namespace: bool = False,
 470    ) -> t.Union[nodes.NSRef, nodes.Name, nodes.Tuple]:
 471        ...
 472
 473    def parse_assign_target(
 474        self,
 475        with_tuple: bool = True,
 476        name_only: bool = False,
 477        extra_end_rules: t.Optional[t.Tuple[str, ...]] = None,
 478        with_namespace: bool = False,
 479    ) -> t.Union[nodes.NSRef, nodes.Name, nodes.Tuple]:
 480        """Parse an assignment target.  As Jinja allows assignments to
 481        tuples, this function can parse all allowed assignment targets.  Per
 482        default assignments to tuples are parsed, that can be disable however
 483        by setting `with_tuple` to `False`.  If only assignments to names are
 484        wanted `name_only` can be set to `True`.  The `extra_end_rules`
 485        parameter is forwarded to the tuple parsing function.  If
 486        `with_namespace` is enabled, a namespace assignment may be parsed.
 487        """
 488        target: nodes.Expr
 489
 490        if with_namespace and self.stream.look().type == "dot":
 491            token = self.stream.expect("name")
 492            next(self.stream)  # dot
 493            attr = self.stream.expect("name")
 494            target = nodes.NSRef(token.value, attr.value, lineno=token.lineno)
 495        elif name_only:
 496            token = self.stream.expect("name")
 497            target = nodes.Name(token.value, "store", lineno=token.lineno)
 498        else:
 499            if with_tuple:
 500                target = self.parse_tuple(
 501                    simplified=True, extra_end_rules=extra_end_rules
 502                )
 503            else:
 504                target = self.parse_primary()
 505
 506            target.set_ctx("store")
 507
 508        if not target.can_assign():
 509            self.fail(
 510                f"can't assign to {type(target).__name__.lower()!r}", target.lineno
 511            )
 512
 513        return target  # type: ignore
 514
 515    def parse_expression(self, with_condexpr: bool = True) -> nodes.Expr:
 516        """Parse an expression.  Per default all expressions are parsed, if
 517        the optional `with_condexpr` parameter is set to `False` conditional
 518        expressions are not parsed.
 519        """
 520        if with_condexpr:
 521            return self.parse_condexpr()
 522        return self.parse_or()
 523
 524    def parse_condexpr(self) -> nodes.Expr:
 525        lineno = self.stream.current.lineno
 526        expr1 = self.parse_or()
 527        expr3: t.Optional[nodes.Expr]
 528
 529        while self.stream.skip_if("name:if"):
 530            expr2 = self.parse_or()
 531            if self.stream.skip_if("name:else"):
 532                expr3 = self.parse_condexpr()
 533            else:
 534                expr3 = None
 535            expr1 = nodes.CondExpr(expr2, expr1, expr3, lineno=lineno)
 536            lineno = self.stream.current.lineno
 537        return expr1
 538
 539    def parse_or(self) -> nodes.Expr:
 540        lineno = self.stream.current.lineno
 541        left = self.parse_and()
 542        while self.stream.skip_if("name:or"):
 543            right = self.parse_and()
 544            left = nodes.Or(left, right, lineno=lineno)
 545            lineno = self.stream.current.lineno
 546        return left
 547
 548    def parse_and(self) -> nodes.Expr:
 549        lineno = self.stream.current.lineno
 550        left = self.parse_not()
 551        while self.stream.skip_if("name:and"):
 552            right = self.parse_not()
 553            left = nodes.And(left, right, lineno=lineno)
 554            lineno = self.stream.current.lineno
 555        return left
 556
 557    def parse_not(self) -> nodes.Expr:
 558        if self.stream.current.test("name:not"):
 559            lineno = next(self.stream).lineno
 560            return nodes.Not(self.parse_not(), lineno=lineno)
 561        return self.parse_compare()
 562
 563    def parse_compare(self) -> nodes.Expr:
 564        lineno = self.stream.current.lineno
 565        expr = self.parse_math1()
 566        ops = []
 567        while True:
 568            token_type = self.stream.current.type
 569            if token_type in _compare_operators:
 570                next(self.stream)
 571                ops.append(nodes.Operand(token_type, self.parse_math1()))
 572            elif self.stream.skip_if("name:in"):
 573                ops.append(nodes.Operand("in", self.parse_math1()))
 574            elif self.stream.current.test("name:not") and self.stream.look().test(
 575                "name:in"
 576            ):
 577                self.stream.skip(2)
 578                ops.append(nodes.Operand("notin", self.parse_math1()))
 579            else:
 580                break
 581            lineno = self.stream.current.lineno
 582        if not ops:
 583            return expr
 584        return nodes.Compare(expr, ops, lineno=lineno)
 585
 586    def parse_math1(self) -> nodes.Expr:
 587        lineno = self.stream.current.lineno
 588        left = self.parse_concat()
 589        while self.stream.current.type in ("add", "sub"):
 590            cls = _math_nodes[self.stream.current.type]
 591            next(self.stream)
 592            right = self.parse_concat()
 593            left = cls(left, right, lineno=lineno)
 594            lineno = self.stream.current.lineno
 595        return left
 596
 597    def parse_concat(self) -> nodes.Expr:
 598        lineno = self.stream.current.lineno
 599        args = [self.parse_math2()]
 600        while self.stream.current.type == "tilde":
 601            next(self.stream)
 602            args.append(self.parse_math2())
 603        if len(args) == 1:
 604            return args[0]
 605        return nodes.Concat(args, lineno=lineno)
 606
 607    def parse_math2(self) -> nodes.Expr:
 608        lineno = self.stream.current.lineno
 609        left = self.parse_pow()
 610        while self.stream.current.type in ("mul", "div", "floordiv", "mod"):
 611            cls = _math_nodes[self.stream.current.type]
 612            next(self.stream)
 613            right = self.parse_pow()
 614            left = cls(left, right, lineno=lineno)
 615            lineno = self.stream.current.lineno
 616        return left
 617
 618    def parse_pow(self) -> nodes.Expr:
 619        lineno = self.stream.current.lineno
 620        left = self.parse_unary()
 621        while self.stream.current.type == "pow":
 622            next(self.stream)
 623            right = self.parse_unary()
 624            left = nodes.Pow(left, right, lineno=lineno)
 625            lineno = self.stream.current.lineno
 626        return left
 627
 628    def parse_unary(self, with_filter: bool = True) -> nodes.Expr:
 629        token_type = self.stream.current.type
 630        lineno = self.stream.current.lineno
 631        node: nodes.Expr
 632
 633        if token_type == "sub":
 634            next(self.stream)
 635            node = nodes.Neg(self.parse_unary(False), lineno=lineno)
 636        elif token_type == "add":
 637            next(self.stream)
 638            node = nodes.Pos(self.parse_unary(False), lineno=lineno)
 639        else:
 640            node = self.parse_primary()
 641        node = self.parse_postfix(node)
 642        if with_filter:
 643            node = self.parse_filter_expr(node)
 644        return node
 645
 646    def parse_primary(self) -> nodes.Expr:
 647        token = self.stream.current
 648        node: nodes.Expr
 649        if token.type == "name":
 650            if token.value in ("true", "false", "True", "False"):
 651                node = nodes.Const(token.value in ("true", "True"), lineno=token.lineno)
 652            elif token.value in ("none", "None"):
 653                node = nodes.Const(None, lineno=token.lineno)
 654            else:
 655                node = nodes.Name(token.value, "load", lineno=token.lineno)
 656            next(self.stream)
 657        elif token.type == "string":
 658            next(self.stream)
 659            buf = [token.value]
 660            lineno = token.lineno
 661            while self.stream.current.type == "string":
 662                buf.append(self.stream.current.value)
 663                next(self.stream)
 664            node = nodes.Const("".join(buf), lineno=lineno)
 665        elif token.type in ("integer", "float"):
 666            next(self.stream)
 667            node = nodes.Const(token.value, lineno=token.lineno)
 668        elif token.type == "lparen":
 669            next(self.stream)
 670            node = self.parse_tuple(explicit_parentheses=True)
 671            self.stream.expect("rparen")
 672        elif token.type == "lbracket":
 673            node = self.parse_list()
 674        elif token.type == "lbrace":
 675            node = self.parse_dict()
 676        else:
 677            self.fail(f"unexpected {describe_token(token)!r}", token.lineno)
 678        return node
 679
 680    def parse_tuple(
 681        self,
 682        simplified: bool = False,
 683        with_condexpr: bool = True,
 684        extra_end_rules: t.Optional[t.Tuple[str, ...]] = None,
 685        explicit_parentheses: bool = False,
 686    ) -> t.Union[nodes.Tuple, nodes.Expr]:
 687        """Works like `parse_expression` but if multiple expressions are
 688        delimited by a comma a :class:`~jinja2.nodes.Tuple` node is created.
 689        This method could also return a regular expression instead of a tuple
 690        if no commas where found.
 691
 692        The default parsing mode is a full tuple.  If `simplified` is `True`
 693        only names and literals are parsed.  The `no_condexpr` parameter is
 694        forwarded to :meth:`parse_expression`.
 695
 696        Because tuples do not require delimiters and may end in a bogus comma
 697        an extra hint is needed that marks the end of a tuple.  For example
 698        for loops support tuples between `for` and `in`.  In that case the
 699        `extra_end_rules` is set to ``['name:in']``.
 700
 701        `explicit_parentheses` is true if the parsing was triggered by an
 702        expression in parentheses.  This is used to figure out if an empty
 703        tuple is a valid expression or not.
 704        """
 705        lineno = self.stream.current.lineno
 706        if simplified:
 707            parse = self.parse_primary
 708        elif with_condexpr:
 709            parse = self.parse_expression
 710        else:
 711
 712            def parse() -> nodes.Expr:
 713                return self.parse_expression(with_condexpr=False)
 714
 715        args: t.List[nodes.Expr] = []
 716        is_tuple = False
 717
 718        while True:
 719            if args:
 720                self.stream.expect("comma")
 721            if self.is_tuple_end(extra_end_rules):
 722                break
 723            args.append(parse())
 724            if self.stream.current.type == "comma":
 725                is_tuple = True
 726            else:
 727                break
 728            lineno = self.stream.current.lineno
 729
 730        if not is_tuple:
 731            if args:
 732                return args[0]
 733
 734            # if we don't have explicit parentheses, an empty tuple is
 735            # not a valid expression.  This would mean nothing (literally
 736            # nothing) in the spot of an expression would be an empty
 737            # tuple.
 738            if not explicit_parentheses:
 739                self.fail(
 740                    "Expected an expression,"
 741                    f" got {describe_token(self.stream.current)!r}"
 742                )
 743
 744        return nodes.Tuple(args, "load", lineno=lineno)
 745
 746    def parse_list(self) -> nodes.List:
 747        token = self.stream.expect("lbracket")
 748        items: t.List[nodes.Expr] = []
 749        while self.stream.current.type != "rbracket":
 750            if items:
 751                self.stream.expect("comma")
 752            if self.stream.current.type == "rbracket":
 753                break
 754            items.append(self.parse_expression())
 755        self.stream.expect("rbracket")
 756        return nodes.List(items, lineno=token.lineno)
 757
 758    def parse_dict(self) -> nodes.Dict:
 759        token = self.stream.expect("lbrace")
 760        items: t.List[nodes.Pair] = []
 761        while self.stream.current.type != "rbrace":
 762            if items:
 763                self.stream.expect("comma")
 764            if self.stream.current.type == "rbrace":
 765                break
 766            key = self.parse_expression()
 767            self.stream.expect("colon")
 768            value = self.parse_expression()
 769            items.append(nodes.Pair(key, value, lineno=key.lineno))
 770        self.stream.expect("rbrace")
 771        return nodes.Dict(items, lineno=token.lineno)
 772
 773    def parse_postfix(self, node: nodes.Expr) -> nodes.Expr:
 774        while True:
 775            token_type = self.stream.current.type
 776            if token_type == "dot" or token_type == "lbracket":
 777                node = self.parse_subscript(node)
 778            # calls are valid both after postfix expressions (getattr
 779            # and getitem) as well as filters and tests
 780            elif token_type == "lparen":
 781                node = self.parse_call(node)
 782            else:
 783                break
 784        return node
 785
 786    def parse_filter_expr(self, node: nodes.Expr) -> nodes.Expr:
 787        while True:
 788            token_type = self.stream.current.type
 789            if token_type == "pipe":
 790                node = self.parse_filter(node)  # type: ignore
 791            elif token_type == "name" and self.stream.current.value == "is":
 792                node = self.parse_test(node)
 793            # calls are valid both after postfix expressions (getattr
 794            # and getitem) as well as filters and tests
 795            elif token_type == "lparen":
 796                node = self.parse_call(node)
 797            else:
 798                break
 799        return node
 800
 801    def parse_subscript(
 802        self, node: nodes.Expr
 803    ) -> t.Union[nodes.Getattr, nodes.Getitem]:
 804        token = next(self.stream)
 805        arg: nodes.Expr
 806
 807        if token.type == "dot":
 808            attr_token = self.stream.current
 809            next(self.stream)
 810            if attr_token.type == "name":
 811                return nodes.Getattr(
 812                    node, attr_token.value, "load", lineno=token.lineno
 813                )
 814            elif attr_token.type != "integer":
 815                self.fail("expected name or number", attr_token.lineno)
 816            arg = nodes.Const(attr_token.value, lineno=attr_token.lineno)
 817            return nodes.Getitem(node, arg, "load", lineno=token.lineno)
 818        if token.type == "lbracket":
 819            args: t.List[nodes.Expr] = []
 820            while self.stream.current.type != "rbracket":
 821                if args:
 822                    self.stream.expect("comma")
 823                args.append(self.parse_subscribed())
 824            self.stream.expect("rbracket")
 825            if len(args) == 1:
 826                arg = args[0]
 827            else:
 828                arg = nodes.Tuple(args, "load", lineno=token.lineno)
 829            return nodes.Getitem(node, arg, "load", lineno=token.lineno)
 830        self.fail("expected subscript expression", token.lineno)
 831
 832    def parse_subscribed(self) -> nodes.Expr:
 833        lineno = self.stream.current.lineno
 834        args: t.List[t.Optional[nodes.Expr]]
 835
 836        if self.stream.current.type == "colon":
 837            next(self.stream)
 838            args = [None]
 839        else:
 840            node = self.parse_expression()
 841            if self.stream.current.type != "colon":
 842                return node
 843            next(self.stream)
 844            args = [node]
 845
 846        if self.stream.current.type == "colon":
 847            args.append(None)
 848        elif self.stream.current.type not in ("rbracket", "comma"):
 849            args.append(self.parse_expression())
 850        else:
 851            args.append(None)
 852
 853        if self.stream.current.type == "colon":
 854            next(self.stream)
 855            if self.stream.current.type not in ("rbracket", "comma"):
 856                args.append(self.parse_expression())
 857            else:
 858                args.append(None)
 859        else:
 860            args.append(None)
 861
 862        return nodes.Slice(lineno=lineno, *args)  # noqa: B026
 863
 864    def parse_call_args(self) -> t.Tuple:
 865        token = self.stream.expect("lparen")
 866        args = []
 867        kwargs = []
 868        dyn_args = None
 869        dyn_kwargs = None
 870        require_comma = False
 871
 872        def ensure(expr: bool) -> None:
 873            if not expr:
 874                self.fail("invalid syntax for function call expression", token.lineno)
 875
 876        while self.stream.current.type != "rparen":
 877            if require_comma:
 878                self.stream.expect("comma")
 879
 880                # support for trailing comma
 881                if self.stream.current.type == "rparen":
 882                    break
 883
 884            if self.stream.current.type == "mul":
 885                ensure(dyn_args is None and dyn_kwargs is None)
 886                next(self.stream)
 887                dyn_args = self.parse_expression()
 888            elif self.stream.current.type == "pow":
 889                ensure(dyn_kwargs is None)
 890                next(self.stream)
 891                dyn_kwargs = self.parse_expression()
 892            else:
 893                if (
 894                    self.stream.current.type == "name"
 895                    and self.stream.look().type == "assign"
 896                ):
 897                    # Parsing a kwarg
 898                    ensure(dyn_kwargs is None)
 899                    key = self.stream.current.value
 900                    self.stream.skip(2)
 901                    value = self.parse_expression()
 902                    kwargs.append(nodes.Keyword(key, value, lineno=value.lineno))
 903                else:
 904                    # Parsing an arg
 905                    ensure(dyn_args is None and dyn_kwargs is None and not kwargs)
 906                    args.append(self.parse_expression())
 907
 908            require_comma = True
 909
 910        self.stream.expect("rparen")
 911        return args, kwargs, dyn_args, dyn_kwargs
 912
 913    def parse_call(self, node: nodes.Expr) -> nodes.Call:
 914        # The lparen will be expected in parse_call_args, but the lineno
 915        # needs to be recorded before the stream is advanced.
 916        token = self.stream.current
 917        args, kwargs, dyn_args, dyn_kwargs = self.parse_call_args()
 918        return nodes.Call(node, args, kwargs, dyn_args, dyn_kwargs, lineno=token.lineno)
 919
 920    def parse_filter(
 921        self, node: t.Optional[nodes.Expr], start_inline: bool = False
 922    ) -> t.Optional[nodes.Expr]:
 923        while self.stream.current.type == "pipe" or start_inline:
 924            if not start_inline:
 925                next(self.stream)
 926            token = self.stream.expect("name")
 927            name = token.value
 928            while self.stream.current.type == "dot":
 929                next(self.stream)
 930                name += "." + self.stream.expect("name").value
 931            if self.stream.current.type == "lparen":
 932                args, kwargs, dyn_args, dyn_kwargs = self.parse_call_args()
 933            else:
 934                args = []
 935                kwargs = []
 936                dyn_args = dyn_kwargs = None
 937            node = nodes.Filter(
 938                node, name, args, kwargs, dyn_args, dyn_kwargs, lineno=token.lineno
 939            )
 940            start_inline = False
 941        return node
 942
 943    def parse_test(self, node: nodes.Expr) -> nodes.Expr:
 944        token = next(self.stream)
 945        if self.stream.current.test("name:not"):
 946            next(self.stream)
 947            negated = True
 948        else:
 949            negated = False
 950        name = self.stream.expect("name").value
 951        while self.stream.current.type == "dot":
 952            next(self.stream)
 953            name += "." + self.stream.expect("name").value
 954        dyn_args = dyn_kwargs = None
 955        kwargs = []
 956        if self.stream.current.type == "lparen":
 957            args, kwargs, dyn_args, dyn_kwargs = self.parse_call_args()
 958        elif self.stream.current.type in {
 959            "name",
 960            "string",
 961            "integer",
 962            "float",
 963            "lparen",
 964            "lbracket",
 965            "lbrace",
 966        } and not self.stream.current.test_any("name:else", "name:or", "name:and"):
 967            if self.stream.current.test("name:is"):
 968                self.fail("You cannot chain multiple tests with is")
 969            arg_node = self.parse_primary()
 970            arg_node = self.parse_postfix(arg_node)
 971            args = [arg_node]
 972        else:
 973            args = []
 974        node = nodes.Test(
 975            node, name, args, kwargs, dyn_args, dyn_kwargs, lineno=token.lineno
 976        )
 977        if negated:
 978            node = nodes.Not(node, lineno=token.lineno)
 979        return node
 980
 981    def subparse(
 982        self, end_tokens: t.Optional[t.Tuple[str, ...]] = None
 983    ) -> t.List[nodes.Node]:
 984        body: t.List[nodes.Node] = []
 985        data_buffer: t.List[nodes.Node] = []
 986        add_data = data_buffer.append
 987
 988        if end_tokens is not None:
 989            self._end_token_stack.append(end_tokens)
 990
 991        def flush_data() -> None:
 992            if data_buffer:
 993                lineno = data_buffer[0].lineno
 994                body.append(nodes.Output(data_buffer[:], lineno=lineno))
 995                del data_buffer[:]
 996
 997        try:
 998            while self.stream:
 999                token = self.stream.current
1000                if token.type == "data":
1001                    if token.value:
1002                        add_data(nodes.TemplateData(token.value, lineno=token.lineno))
1003                    next(self.stream)
1004                elif token.type == "variable_begin":
1005                    next(self.stream)
1006                    add_data(self.parse_tuple(with_condexpr=True))
1007                    self.stream.expect("variable_end")
1008                elif token.type == "block_begin":
1009                    flush_data()
1010                    next(self.stream)
1011                    if end_tokens is not None and self.stream.current.test_any(
1012                        *end_tokens
1013                    ):
1014                        return body
1015                    rv = self.parse_statement()
1016                    if isinstance(rv, list):
1017                        body.extend(rv)
1018                    else:
1019                        body.append(rv)
1020                    self.stream.expect("block_end")
1021                else:
1022                    raise AssertionError("internal parsing error")
1023
1024            flush_data()
1025        finally:
1026            if end_tokens is not None:
1027                self._end_token_stack.pop()
1028        return body
1029
1030    def parse(self) -> nodes.Template:
1031        """Parse the whole template into a `Template` node."""
1032        result = nodes.Template(self.subparse(), lineno=1)
1033        result.set_environment(self.environment)
1034        return result