jinja2.parser

   1"""Parse tokens from the lexer into nodes for the compiler."""
   2
   3import typing
   4import typing as t
   5
   6from . import nodes
   7from .exceptions import TemplateAssertionError
   8from .exceptions import TemplateSyntaxError
   9from .lexer import describe_token
  10from .lexer import describe_token_expr
  11
  12if t.TYPE_CHECKING:
  13    import typing_extensions as te
  14
  15    from .environment import Environment
  16
  17_ImportInclude = t.TypeVar("_ImportInclude", nodes.Import, nodes.Include)
  18_MacroCall = t.TypeVar("_MacroCall", nodes.Macro, nodes.CallBlock)
  19
  20_statement_keywords = frozenset(
  21    [
  22        "for",
  23        "if",
  24        "block",
  25        "extends",
  26        "print",
  27        "macro",
  28        "include",
  29        "from",
  30        "import",
  31        "set",
  32        "with",
  33        "autoescape",
  34    ]
  35)
  36_compare_operators = frozenset(["eq", "ne", "lt", "lteq", "gt", "gteq"])
  37
  38_math_nodes: t.Dict[str, t.Type[nodes.Expr]] = {
  39    "add": nodes.Add,
  40    "sub": nodes.Sub,
  41    "mul": nodes.Mul,
  42    "div": nodes.Div,
  43    "floordiv": nodes.FloorDiv,
  44    "mod": nodes.Mod,
  45}
  46
  47
  48class Parser:
  49    """This is the central parsing class Jinja uses.  It's passed to
  50    extensions and can be used to parse expressions or statements.
  51    """
  52
  53    def __init__(
  54        self,
  55        environment: "Environment",
  56        source: str,
  57        name: t.Optional[str] = None,
  58        filename: t.Optional[str] = None,
  59        state: t.Optional[str] = None,
  60    ) -> None:
  61        self.environment = environment
  62        self.stream = environment._tokenize(source, name, filename, state)
  63        self.name = name
  64        self.filename = filename
  65        self.closed = False
  66        self.extensions: t.Dict[
  67            str, t.Callable[["Parser"], t.Union[nodes.Node, t.List[nodes.Node]]]
  68        ] = {}
  69        for extension in environment.iter_extensions():
  70            for tag in extension.tags:
  71                self.extensions[tag] = extension.parse
  72        self._last_identifier = 0
  73        self._tag_stack: t.List[str] = []
  74        self._end_token_stack: t.List[t.Tuple[str, ...]] = []
  75
  76    def fail(
  77        self,
  78        msg: str,
  79        lineno: t.Optional[int] = None,
  80        exc: t.Type[TemplateSyntaxError] = TemplateSyntaxError,
  81    ) -> "te.NoReturn":
  82        """Convenience method that raises `exc` with the message, passed
  83        line number or last line number as well as the current name and
  84        filename.
  85        """
  86        if lineno is None:
  87            lineno = self.stream.current.lineno
  88        raise exc(msg, lineno, self.name, self.filename)
  89
  90    def _fail_ut_eof(
  91        self,
  92        name: t.Optional[str],
  93        end_token_stack: t.List[t.Tuple[str, ...]],
  94        lineno: t.Optional[int],
  95    ) -> "te.NoReturn":
  96        expected: t.Set[str] = set()
  97        for exprs in end_token_stack:
  98            expected.update(map(describe_token_expr, exprs))
  99        if end_token_stack:
 100            currently_looking: t.Optional[str] = " or ".join(
 101                map(repr, map(describe_token_expr, end_token_stack[-1]))
 102            )
 103        else:
 104            currently_looking = None
 105
 106        if name is None:
 107            message = ["Unexpected end of template."]
 108        else:
 109            message = [f"Encountered unknown tag {name!r}."]
 110
 111        if currently_looking:
 112            if name is not None and name in expected:
 113                message.append(
 114                    "You probably made a nesting mistake. Jinja is expecting this tag,"
 115                    f" but currently looking for {currently_looking}."
 116                )
 117            else:
 118                message.append(
 119                    f"Jinja was looking for the following tags: {currently_looking}."
 120                )
 121
 122        if self._tag_stack:
 123            message.append(
 124                "The innermost block that needs to be closed is"
 125                f" {self._tag_stack[-1]!r}."
 126            )
 127
 128        self.fail(" ".join(message), lineno)
 129
 130    def fail_unknown_tag(
 131        self, name: str, lineno: t.Optional[int] = None
 132    ) -> "te.NoReturn":
 133        """Called if the parser encounters an unknown tag.  Tries to fail
 134        with a human readable error message that could help to identify
 135        the problem.
 136        """
 137        self._fail_ut_eof(name, self._end_token_stack, lineno)
 138
 139    def fail_eof(
 140        self,
 141        end_tokens: t.Optional[t.Tuple[str, ...]] = None,
 142        lineno: t.Optional[int] = None,
 143    ) -> "te.NoReturn":
 144        """Like fail_unknown_tag but for end of template situations."""
 145        stack = list(self._end_token_stack)
 146        if end_tokens is not None:
 147            stack.append(end_tokens)
 148        self._fail_ut_eof(None, stack, lineno)
 149
 150    def is_tuple_end(
 151        self, extra_end_rules: t.Optional[t.Tuple[str, ...]] = None
 152    ) -> bool:
 153        """Are we at the end of a tuple?"""
 154        if self.stream.current.type in ("variable_end", "block_end", "rparen"):
 155            return True
 156        elif extra_end_rules is not None:
 157            return self.stream.current.test_any(extra_end_rules)  # type: ignore
 158        return False
 159
 160    def free_identifier(self, lineno: t.Optional[int] = None) -> nodes.InternalName:
 161        """Return a new free identifier as :class:`~jinja2.nodes.InternalName`."""
 162        self._last_identifier += 1
 163        rv = object.__new__(nodes.InternalName)
 164        nodes.Node.__init__(rv, f"fi{self._last_identifier}", lineno=lineno)
 165        return rv
 166
 167    def parse_statement(self) -> t.Union[nodes.Node, t.List[nodes.Node]]:
 168        """Parse a single statement."""
 169        token = self.stream.current
 170        if token.type != "name":
 171            self.fail("tag name expected", token.lineno)
 172        self._tag_stack.append(token.value)
 173        pop_tag = True
 174        try:
 175            if token.value in _statement_keywords:
 176                f = getattr(self, f"parse_{self.stream.current.value}")
 177                return f()  # type: ignore
 178            if token.value == "call":
 179                return self.parse_call_block()
 180            if token.value == "filter":
 181                return self.parse_filter_block()
 182            ext = self.extensions.get(token.value)
 183            if ext is not None:
 184                return ext(self)
 185
 186            # did not work out, remove the token we pushed by accident
 187            # from the stack so that the unknown tag fail function can
 188            # produce a proper error message.
 189            self._tag_stack.pop()
 190            pop_tag = False
 191            self.fail_unknown_tag(token.value, token.lineno)
 192        finally:
 193            if pop_tag:
 194                self._tag_stack.pop()
 195
 196    def parse_statements(
 197        self, end_tokens: t.Tuple[str, ...], drop_needle: bool = False
 198    ) -> t.List[nodes.Node]:
 199        """Parse multiple statements into a list until one of the end tokens
 200        is reached.  This is used to parse the body of statements as it also
 201        parses template data if appropriate.  The parser checks first if the
 202        current token is a colon and skips it if there is one.  Then it checks
 203        for the block end and parses until if one of the `end_tokens` is
 204        reached.  Per default the active token in the stream at the end of
 205        the call is the matched end token.  If this is not wanted `drop_needle`
 206        can be set to `True` and the end token is removed.
 207        """
 208        # the first token may be a colon for python compatibility
 209        self.stream.skip_if("colon")
 210
 211        # in the future it would be possible to add whole code sections
 212        # by adding some sort of end of statement token and parsing those here.
 213        self.stream.expect("block_end")
 214        result = self.subparse(end_tokens)
 215
 216        # we reached the end of the template too early, the subparser
 217        # does not check for this, so we do that now
 218        if self.stream.current.type == "eof":
 219            self.fail_eof(end_tokens)
 220
 221        if drop_needle:
 222            next(self.stream)
 223        return result
 224
 225    def parse_set(self) -> t.Union[nodes.Assign, nodes.AssignBlock]:
 226        """Parse an assign statement."""
 227        lineno = next(self.stream).lineno
 228        target = self.parse_assign_target(with_namespace=True)
 229        if self.stream.skip_if("assign"):
 230            expr = self.parse_tuple()
 231            return nodes.Assign(target, expr, lineno=lineno)
 232        filter_node = self.parse_filter(None)
 233        body = self.parse_statements(("name:endset",), drop_needle=True)
 234        return nodes.AssignBlock(target, filter_node, body, lineno=lineno)
 235
 236    def parse_for(self) -> nodes.For:
 237        """Parse a for loop."""
 238        lineno = self.stream.expect("name:for").lineno
 239        target = self.parse_assign_target(extra_end_rules=("name:in",))
 240        self.stream.expect("name:in")
 241        iter = self.parse_tuple(
 242            with_condexpr=False, extra_end_rules=("name:recursive",)
 243        )
 244        test = None
 245        if self.stream.skip_if("name:if"):
 246            test = self.parse_expression()
 247        recursive = self.stream.skip_if("name:recursive")
 248        body = self.parse_statements(("name:endfor", "name:else"))
 249        if next(self.stream).value == "endfor":
 250            else_ = []
 251        else:
 252            else_ = self.parse_statements(("name:endfor",), drop_needle=True)
 253        return nodes.For(target, iter, body, else_, test, recursive, lineno=lineno)
 254
 255    def parse_if(self) -> nodes.If:
 256        """Parse an if construct."""
 257        node = result = nodes.If(lineno=self.stream.expect("name:if").lineno)
 258        while True:
 259            node.test = self.parse_tuple(with_condexpr=False)
 260            node.body = self.parse_statements(("name:elif", "name:else", "name:endif"))
 261            node.elif_ = []
 262            node.else_ = []
 263            token = next(self.stream)
 264            if token.test("name:elif"):
 265                node = nodes.If(lineno=self.stream.current.lineno)
 266                result.elif_.append(node)
 267                continue
 268            elif token.test("name:else"):
 269                result.else_ = self.parse_statements(("name:endif",), drop_needle=True)
 270            break
 271        return result
 272
 273    def parse_with(self) -> nodes.With:
 274        node = nodes.With(lineno=next(self.stream).lineno)
 275        targets: t.List[nodes.Expr] = []
 276        values: t.List[nodes.Expr] = []
 277        while self.stream.current.type != "block_end":
 278            if targets:
 279                self.stream.expect("comma")
 280            target = self.parse_assign_target()
 281            target.set_ctx("param")
 282            targets.append(target)
 283            self.stream.expect("assign")
 284            values.append(self.parse_expression())
 285        node.targets = targets
 286        node.values = values
 287        node.body = self.parse_statements(("name:endwith",), drop_needle=True)
 288        return node
 289
 290    def parse_autoescape(self) -> nodes.Scope:
 291        node = nodes.ScopedEvalContextModifier(lineno=next(self.stream).lineno)
 292        node.options = [nodes.Keyword("autoescape", self.parse_expression())]
 293        node.body = self.parse_statements(("name:endautoescape",), drop_needle=True)
 294        return nodes.Scope([node])
 295
 296    def parse_block(self) -> nodes.Block:
 297        node = nodes.Block(lineno=next(self.stream).lineno)
 298        node.name = self.stream.expect("name").value
 299        node.scoped = self.stream.skip_if("name:scoped")
 300        node.required = self.stream.skip_if("name:required")
 301
 302        # common problem people encounter when switching from django
 303        # to jinja.  we do not support hyphens in block names, so let's
 304        # raise a nicer error message in that case.
 305        if self.stream.current.type == "sub":
 306            self.fail(
 307                "Block names in Jinja have to be valid Python identifiers and may not"
 308                " contain hyphens, use an underscore instead."
 309            )
 310
 311        node.body = self.parse_statements(("name:endblock",), drop_needle=True)
 312
 313        # enforce that required blocks only contain whitespace or comments
 314        # by asserting that the body, if not empty, is just TemplateData nodes
 315        # with whitespace data
 316        if node.required:
 317            for body_node in node.body:
 318                if not isinstance(body_node, nodes.Output) or any(
 319                    not isinstance(output_node, nodes.TemplateData)
 320                    or not output_node.data.isspace()
 321                    for output_node in body_node.nodes
 322                ):
 323                    self.fail("Required blocks can only contain comments or whitespace")
 324
 325        self.stream.skip_if("name:" + node.name)
 326        return node
 327
 328    def parse_extends(self) -> nodes.Extends:
 329        node = nodes.Extends(lineno=next(self.stream).lineno)
 330        node.template = self.parse_expression()
 331        return node
 332
 333    def parse_import_context(
 334        self, node: _ImportInclude, default: bool
 335    ) -> _ImportInclude:
 336        if self.stream.current.test_any(
 337            "name:with", "name:without"
 338        ) and self.stream.look().test("name:context"):
 339            node.with_context = next(self.stream).value == "with"
 340            self.stream.skip()
 341        else:
 342            node.with_context = default
 343        return node
 344
 345    def parse_include(self) -> nodes.Include:
 346        node = nodes.Include(lineno=next(self.stream).lineno)
 347        node.template = self.parse_expression()
 348        if self.stream.current.test("name:ignore") and self.stream.look().test(
 349            "name:missing"
 350        ):
 351            node.ignore_missing = True
 352            self.stream.skip(2)
 353        else:
 354            node.ignore_missing = False
 355        return self.parse_import_context(node, True)
 356
 357    def parse_import(self) -> nodes.Import:
 358        node = nodes.Import(lineno=next(self.stream).lineno)
 359        node.template = self.parse_expression()
 360        self.stream.expect("name:as")
 361        node.target = self.parse_assign_target(name_only=True).name
 362        return self.parse_import_context(node, False)
 363
 364    def parse_from(self) -> nodes.FromImport:
 365        node = nodes.FromImport(lineno=next(self.stream).lineno)
 366        node.template = self.parse_expression()
 367        self.stream.expect("name:import")
 368        node.names = []
 369
 370        def parse_context() -> bool:
 371            if self.stream.current.value in {
 372                "with",
 373                "without",
 374            } and self.stream.look().test("name:context"):
 375                node.with_context = next(self.stream).value == "with"
 376                self.stream.skip()
 377                return True
 378            return False
 379
 380        while True:
 381            if node.names:
 382                self.stream.expect("comma")
 383            if self.stream.current.type == "name":
 384                if parse_context():
 385                    break
 386                target = self.parse_assign_target(name_only=True)
 387                if target.name.startswith("_"):
 388                    self.fail(
 389                        "names starting with an underline can not be imported",
 390                        target.lineno,
 391                        exc=TemplateAssertionError,
 392                    )
 393                if self.stream.skip_if("name:as"):
 394                    alias = self.parse_assign_target(name_only=True)
 395                    node.names.append((target.name, alias.name))
 396                else:
 397                    node.names.append(target.name)
 398                if parse_context() or self.stream.current.type != "comma":
 399                    break
 400            else:
 401                self.stream.expect("name")
 402        if not hasattr(node, "with_context"):
 403            node.with_context = False
 404        return node
 405
 406    def parse_signature(self, node: _MacroCall) -> None:
 407        args = node.args = []
 408        defaults = node.defaults = []
 409        self.stream.expect("lparen")
 410        while self.stream.current.type != "rparen":
 411            if args:
 412                self.stream.expect("comma")
 413            arg = self.parse_assign_target(name_only=True)
 414            arg.set_ctx("param")
 415            if self.stream.skip_if("assign"):
 416                defaults.append(self.parse_expression())
 417            elif defaults:
 418                self.fail("non-default argument follows default argument")
 419            args.append(arg)
 420        self.stream.expect("rparen")
 421
 422    def parse_call_block(self) -> nodes.CallBlock:
 423        node = nodes.CallBlock(lineno=next(self.stream).lineno)
 424        if self.stream.current.type == "lparen":
 425            self.parse_signature(node)
 426        else:
 427            node.args = []
 428            node.defaults = []
 429
 430        call_node = self.parse_expression()
 431        if not isinstance(call_node, nodes.Call):
 432            self.fail("expected call", node.lineno)
 433        node.call = call_node
 434        node.body = self.parse_statements(("name:endcall",), drop_needle=True)
 435        return node
 436
 437    def parse_filter_block(self) -> nodes.FilterBlock:
 438        node = nodes.FilterBlock(lineno=next(self.stream).lineno)
 439        node.filter = self.parse_filter(None, start_inline=True)  # type: ignore
 440        node.body = self.parse_statements(("name:endfilter",), drop_needle=True)
 441        return node
 442
 443    def parse_macro(self) -> nodes.Macro:
 444        node = nodes.Macro(lineno=next(self.stream).lineno)
 445        node.name = self.parse_assign_target(name_only=True).name
 446        self.parse_signature(node)
 447        node.body = self.parse_statements(("name:endmacro",), drop_needle=True)
 448        return node
 449
 450    def parse_print(self) -> nodes.Output:
 451        node = nodes.Output(lineno=next(self.stream).lineno)
 452        node.nodes = []
 453        while self.stream.current.type != "block_end":
 454            if node.nodes:
 455                self.stream.expect("comma")
 456            node.nodes.append(self.parse_expression())
 457        return node
 458
 459    @typing.overload
 460    def parse_assign_target(
 461        self, with_tuple: bool = ..., name_only: "te.Literal[True]" = ...
 462    ) -> nodes.Name: ...
 463
 464    @typing.overload
 465    def parse_assign_target(
 466        self,
 467        with_tuple: bool = True,
 468        name_only: bool = False,
 469        extra_end_rules: t.Optional[t.Tuple[str, ...]] = None,
 470        with_namespace: bool = False,
 471    ) -> t.Union[nodes.NSRef, nodes.Name, nodes.Tuple]: ...
 472
 473    def parse_assign_target(
 474        self,
 475        with_tuple: bool = True,
 476        name_only: bool = False,
 477        extra_end_rules: t.Optional[t.Tuple[str, ...]] = None,
 478        with_namespace: bool = False,
 479    ) -> t.Union[nodes.NSRef, nodes.Name, nodes.Tuple]:
 480        """Parse an assignment target.  As Jinja allows assignments to
 481        tuples, this function can parse all allowed assignment targets.  Per
 482        default assignments to tuples are parsed, that can be disable however
 483        by setting `with_tuple` to `False`.  If only assignments to names are
 484        wanted `name_only` can be set to `True`.  The `extra_end_rules`
 485        parameter is forwarded to the tuple parsing function.  If
 486        `with_namespace` is enabled, a namespace assignment may be parsed.
 487        """
 488        target: nodes.Expr
 489
 490        if with_namespace and self.stream.look().type == "dot":
 491            token = self.stream.expect("name")
 492            next(self.stream)  # dot
 493            attr = self.stream.expect("name")
 494            target = nodes.NSRef(token.value, attr.value, lineno=token.lineno)
 495        elif name_only:
 496            token = self.stream.expect("name")
 497            target = nodes.Name(token.value, "store", lineno=token.lineno)
 498        else:
 499            if with_tuple:
 500                target = self.parse_tuple(
 501                    simplified=True, extra_end_rules=extra_end_rules
 502                )
 503            else:
 504                target = self.parse_primary()
 505
 506            target.set_ctx("store")
 507
 508        if not target.can_assign():
 509            self.fail(
 510                f"can't assign to {type(target).__name__.lower()!r}", target.lineno
 511            )
 512
 513        return target  # type: ignore
 514
 515    def parse_expression(self, with_condexpr: bool = True) -> nodes.Expr:
 516        """Parse an expression.  Per default all expressions are parsed, if
 517        the optional `with_condexpr` parameter is set to `False` conditional
 518        expressions are not parsed.
 519        """
 520        if with_condexpr:
 521            return self.parse_condexpr()
 522        return self.parse_or()
 523
 524    def parse_condexpr(self) -> nodes.Expr:
 525        lineno = self.stream.current.lineno
 526        expr1 = self.parse_or()
 527        expr3: t.Optional[nodes.Expr]
 528
 529        while self.stream.skip_if("name:if"):
 530            expr2 = self.parse_or()
 531            if self.stream.skip_if("name:else"):
 532                expr3 = self.parse_condexpr()
 533            else:
 534                expr3 = None
 535            expr1 = nodes.CondExpr(expr2, expr1, expr3, lineno=lineno)
 536            lineno = self.stream.current.lineno
 537        return expr1
 538
 539    def parse_or(self) -> nodes.Expr:
 540        lineno = self.stream.current.lineno
 541        left = self.parse_and()
 542        while self.stream.skip_if("name:or"):
 543            right = self.parse_and()
 544            left = nodes.Or(left, right, lineno=lineno)
 545            lineno = self.stream.current.lineno
 546        return left
 547
 548    def parse_and(self) -> nodes.Expr:
 549        lineno = self.stream.current.lineno
 550        left = self.parse_not()
 551        while self.stream.skip_if("name:and"):
 552            right = self.parse_not()
 553            left = nodes.And(left, right, lineno=lineno)
 554            lineno = self.stream.current.lineno
 555        return left
 556
 557    def parse_not(self) -> nodes.Expr:
 558        if self.stream.current.test("name:not"):
 559            lineno = next(self.stream).lineno
 560            return nodes.Not(self.parse_not(), lineno=lineno)
 561        return self.parse_compare()
 562
 563    def parse_compare(self) -> nodes.Expr:
 564        lineno = self.stream.current.lineno
 565        expr = self.parse_math1()
 566        ops = []
 567        while True:
 568            token_type = self.stream.current.type
 569            if token_type in _compare_operators:
 570                next(self.stream)
 571                ops.append(nodes.Operand(token_type, self.parse_math1()))
 572            elif self.stream.skip_if("name:in"):
 573                ops.append(nodes.Operand("in", self.parse_math1()))
 574            elif self.stream.current.test("name:not") and self.stream.look().test(
 575                "name:in"
 576            ):
 577                self.stream.skip(2)
 578                ops.append(nodes.Operand("notin", self.parse_math1()))
 579            else:
 580                break
 581            lineno = self.stream.current.lineno
 582        if not ops:
 583            return expr
 584        return nodes.Compare(expr, ops, lineno=lineno)
 585
 586    def parse_math1(self) -> nodes.Expr:
 587        lineno = self.stream.current.lineno
 588        left = self.parse_concat()
 589        while self.stream.current.type in ("add", "sub"):
 590            cls = _math_nodes[self.stream.current.type]
 591            next(self.stream)
 592            right = self.parse_concat()
 593            left = cls(left, right, lineno=lineno)
 594            lineno = self.stream.current.lineno
 595        return left
 596
 597    def parse_concat(self) -> nodes.Expr:
 598        lineno = self.stream.current.lineno
 599        args = [self.parse_math2()]
 600        while self.stream.current.type == "tilde":
 601            next(self.stream)
 602            args.append(self.parse_math2())
 603        if len(args) == 1:
 604            return args[0]
 605        return nodes.Concat(args, lineno=lineno)
 606
 607    def parse_math2(self) -> nodes.Expr:
 608        lineno = self.stream.current.lineno
 609        left = self.parse_pow()
 610        while self.stream.current.type in ("mul", "div", "floordiv", "mod"):
 611            cls = _math_nodes[self.stream.current.type]
 612            next(self.stream)
 613            right = self.parse_pow()
 614            left = cls(left, right, lineno=lineno)
 615            lineno = self.stream.current.lineno
 616        return left
 617
 618    def parse_pow(self) -> nodes.Expr:
 619        lineno = self.stream.current.lineno
 620        left = self.parse_unary()
 621        while self.stream.current.type == "pow":
 622            next(self.stream)
 623            right = self.parse_unary()
 624            left = nodes.Pow(left, right, lineno=lineno)
 625            lineno = self.stream.current.lineno
 626        return left
 627
 628    def parse_unary(self, with_filter: bool = True) -> nodes.Expr:
 629        token_type = self.stream.current.type
 630        lineno = self.stream.current.lineno
 631        node: nodes.Expr
 632
 633        if token_type == "sub":
 634            next(self.stream)
 635            node = nodes.Neg(self.parse_unary(False), lineno=lineno)
 636        elif token_type == "add":
 637            next(self.stream)
 638            node = nodes.Pos(self.parse_unary(False), lineno=lineno)
 639        else:
 640            node = self.parse_primary()
 641        node = self.parse_postfix(node)
 642        if with_filter:
 643            node = self.parse_filter_expr(node)
 644        return node
 645
 646    def parse_primary(self) -> nodes.Expr:
 647        token = self.stream.current
 648        node: nodes.Expr
 649        if token.type == "name":
 650            if token.value in ("true", "false", "True", "False"):
 651                node = nodes.Const(token.value in ("true", "True"), lineno=token.lineno)
 652            elif token.value in ("none", "None"):
 653                node = nodes.Const(None, lineno=token.lineno)
 654            else:
 655                node = nodes.Name(token.value, "load", lineno=token.lineno)
 656            next(self.stream)
 657        elif token.type == "string":
 658            next(self.stream)
 659            buf = [token.value]
 660            lineno = token.lineno
 661            while self.stream.current.type == "string":
 662                buf.append(self.stream.current.value)
 663                next(self.stream)
 664            node = nodes.Const("".join(buf), lineno=lineno)
 665        elif token.type in ("integer", "float"):
 666            next(self.stream)
 667            node = nodes.Const(token.value, lineno=token.lineno)
 668        elif token.type == "lparen":
 669            next(self.stream)
 670            node = self.parse_tuple(explicit_parentheses=True)
 671            self.stream.expect("rparen")
 672        elif token.type == "lbracket":
 673            node = self.parse_list()
 674        elif token.type == "lbrace":
 675            node = self.parse_dict()
 676        else:
 677            self.fail(f"unexpected {describe_token(token)!r}", token.lineno)
 678        return node
 679
 680    def parse_tuple(
 681        self,
 682        simplified: bool = False,
 683        with_condexpr: bool = True,
 684        extra_end_rules: t.Optional[t.Tuple[str, ...]] = None,
 685        explicit_parentheses: bool = False,
 686    ) -> t.Union[nodes.Tuple, nodes.Expr]:
 687        """Works like `parse_expression` but if multiple expressions are
 688        delimited by a comma a :class:`~jinja2.nodes.Tuple` node is created.
 689        This method could also return a regular expression instead of a tuple
 690        if no commas where found.
 691
 692        The default parsing mode is a full tuple.  If `simplified` is `True`
 693        only names and literals are parsed.  The `no_condexpr` parameter is
 694        forwarded to :meth:`parse_expression`.
 695
 696        Because tuples do not require delimiters and may end in a bogus comma
 697        an extra hint is needed that marks the end of a tuple.  For example
 698        for loops support tuples between `for` and `in`.  In that case the
 699        `extra_end_rules` is set to ``['name:in']``.
 700
 701        `explicit_parentheses` is true if the parsing was triggered by an
 702        expression in parentheses.  This is used to figure out if an empty
 703        tuple is a valid expression or not.
 704        """
 705        lineno = self.stream.current.lineno
 706        if simplified:
 707            parse = self.parse_primary
 708        elif with_condexpr:
 709            parse = self.parse_expression
 710        else:
 711
 712            def parse() -> nodes.Expr:
 713                return self.parse_expression(with_condexpr=False)
 714
 715        args: t.List[nodes.Expr] = []
 716        is_tuple = False
 717
 718        while True:
 719            if args:
 720                self.stream.expect("comma")
 721            if self.is_tuple_end(extra_end_rules):
 722                break
 723            args.append(parse())
 724            if self.stream.current.type == "comma":
 725                is_tuple = True
 726            else:
 727                break
 728            lineno = self.stream.current.lineno
 729
 730        if not is_tuple:
 731            if args:
 732                return args[0]
 733
 734            # if we don't have explicit parentheses, an empty tuple is
 735            # not a valid expression.  This would mean nothing (literally
 736            # nothing) in the spot of an expression would be an empty
 737            # tuple.
 738            if not explicit_parentheses:
 739                self.fail(
 740                    "Expected an expression,"
 741                    f" got {describe_token(self.stream.current)!r}"
 742                )
 743
 744        return nodes.Tuple(args, "load", lineno=lineno)
 745
 746    def parse_list(self) -> nodes.List:
 747        token = self.stream.expect("lbracket")
 748        items: t.List[nodes.Expr] = []
 749        while self.stream.current.type != "rbracket":
 750            if items:
 751                self.stream.expect("comma")
 752            if self.stream.current.type == "rbracket":
 753                break
 754            items.append(self.parse_expression())
 755        self.stream.expect("rbracket")
 756        return nodes.List(items, lineno=token.lineno)
 757
 758    def parse_dict(self) -> nodes.Dict:
 759        token = self.stream.expect("lbrace")
 760        items: t.List[nodes.Pair] = []
 761        while self.stream.current.type != "rbrace":
 762            if items:
 763                self.stream.expect("comma")
 764            if self.stream.current.type == "rbrace":
 765                break
 766            key = self.parse_expression()
 767            self.stream.expect("colon")
 768            value = self.parse_expression()
 769            items.append(nodes.Pair(key, value, lineno=key.lineno))
 770        self.stream.expect("rbrace")
 771        return nodes.Dict(items, lineno=token.lineno)
 772
 773    def parse_postfix(self, node: nodes.Expr) -> nodes.Expr:
 774        while True:
 775            token_type = self.stream.current.type
 776            if token_type == "dot" or token_type == "lbracket":
 777                node = self.parse_subscript(node)
 778            # calls are valid both after postfix expressions (getattr
 779            # and getitem) as well as filters and tests
 780            elif token_type == "lparen":
 781                node = self.parse_call(node)
 782            else:
 783                break
 784        return node
 785
 786    def parse_filter_expr(self, node: nodes.Expr) -> nodes.Expr:
 787        while True:
 788            token_type = self.stream.current.type
 789            if token_type == "pipe":
 790                node = self.parse_filter(node)  # type: ignore
 791            elif token_type == "name" and self.stream.current.value == "is":
 792                node = self.parse_test(node)
 793            # calls are valid both after postfix expressions (getattr
 794            # and getitem) as well as filters and tests
 795            elif token_type == "lparen":
 796                node = self.parse_call(node)
 797            else:
 798                break
 799        return node
 800
 801    def parse_subscript(
 802        self, node: nodes.Expr
 803    ) -> t.Union[nodes.Getattr, nodes.Getitem]:
 804        token = next(self.stream)
 805        arg: nodes.Expr
 806
 807        if token.type == "dot":
 808            attr_token = self.stream.current
 809            next(self.stream)
 810            if attr_token.type == "name":
 811                return nodes.Getattr(
 812                    node, attr_token.value, "load", lineno=token.lineno
 813                )
 814            elif attr_token.type != "integer":
 815                self.fail("expected name or number", attr_token.lineno)
 816            arg = nodes.Const(attr_token.value, lineno=attr_token.lineno)
 817            return nodes.Getitem(node, arg, "load", lineno=token.lineno)
 818        if token.type == "lbracket":
 819            args: t.List[nodes.Expr] = []
 820            while self.stream.current.type != "rbracket":
 821                if args:
 822                    self.stream.expect("comma")
 823                args.append(self.parse_subscribed())
 824            self.stream.expect("rbracket")
 825            if len(args) == 1:
 826                arg = args[0]
 827            else:
 828                arg = nodes.Tuple(args, "load", lineno=token.lineno)
 829            return nodes.Getitem(node, arg, "load", lineno=token.lineno)
 830        self.fail("expected subscript expression", token.lineno)
 831
 832    def parse_subscribed(self) -> nodes.Expr:
 833        lineno = self.stream.current.lineno
 834        args: t.List[t.Optional[nodes.Expr]]
 835
 836        if self.stream.current.type == "colon":
 837            next(self.stream)
 838            args = [None]
 839        else:
 840            node = self.parse_expression()
 841            if self.stream.current.type != "colon":
 842                return node
 843            next(self.stream)
 844            args = [node]
 845
 846        if self.stream.current.type == "colon":
 847            args.append(None)
 848        elif self.stream.current.type not in ("rbracket", "comma"):
 849            args.append(self.parse_expression())
 850        else:
 851            args.append(None)
 852
 853        if self.stream.current.type == "colon":
 854            next(self.stream)
 855            if self.stream.current.type not in ("rbracket", "comma"):
 856                args.append(self.parse_expression())
 857            else:
 858                args.append(None)
 859        else:
 860            args.append(None)
 861
 862        return nodes.Slice(lineno=lineno, *args)  # noqa: B026
 863
 864    def parse_call_args(
 865        self,
 866    ) -> t.Tuple[
 867        t.List[nodes.Expr],
 868        t.List[nodes.Keyword],
 869        t.Optional[nodes.Expr],
 870        t.Optional[nodes.Expr],
 871    ]:
 872        token = self.stream.expect("lparen")
 873        args = []
 874        kwargs = []
 875        dyn_args = None
 876        dyn_kwargs = None
 877        require_comma = False
 878
 879        def ensure(expr: bool) -> None:
 880            if not expr:
 881                self.fail("invalid syntax for function call expression", token.lineno)
 882
 883        while self.stream.current.type != "rparen":
 884            if require_comma:
 885                self.stream.expect("comma")
 886
 887                # support for trailing comma
 888                if self.stream.current.type == "rparen":
 889                    break
 890
 891            if self.stream.current.type == "mul":
 892                ensure(dyn_args is None and dyn_kwargs is None)
 893                next(self.stream)
 894                dyn_args = self.parse_expression()
 895            elif self.stream.current.type == "pow":
 896                ensure(dyn_kwargs is None)
 897                next(self.stream)
 898                dyn_kwargs = self.parse_expression()
 899            else:
 900                if (
 901                    self.stream.current.type == "name"
 902                    and self.stream.look().type == "assign"
 903                ):
 904                    # Parsing a kwarg
 905                    ensure(dyn_kwargs is None)
 906                    key = self.stream.current.value
 907                    self.stream.skip(2)
 908                    value = self.parse_expression()
 909                    kwargs.append(nodes.Keyword(key, value, lineno=value.lineno))
 910                else:
 911                    # Parsing an arg
 912                    ensure(dyn_args is None and dyn_kwargs is None and not kwargs)
 913                    args.append(self.parse_expression())
 914
 915            require_comma = True
 916
 917        self.stream.expect("rparen")
 918        return args, kwargs, dyn_args, dyn_kwargs
 919
 920    def parse_call(self, node: nodes.Expr) -> nodes.Call:
 921        # The lparen will be expected in parse_call_args, but the lineno
 922        # needs to be recorded before the stream is advanced.
 923        token = self.stream.current
 924        args, kwargs, dyn_args, dyn_kwargs = self.parse_call_args()
 925        return nodes.Call(node, args, kwargs, dyn_args, dyn_kwargs, lineno=token.lineno)
 926
 927    def parse_filter(
 928        self, node: t.Optional[nodes.Expr], start_inline: bool = False
 929    ) -> t.Optional[nodes.Expr]:
 930        while self.stream.current.type == "pipe" or start_inline:
 931            if not start_inline:
 932                next(self.stream)
 933            token = self.stream.expect("name")
 934            name = token.value
 935            while self.stream.current.type == "dot":
 936                next(self.stream)
 937                name += "." + self.stream.expect("name").value
 938            if self.stream.current.type == "lparen":
 939                args, kwargs, dyn_args, dyn_kwargs = self.parse_call_args()
 940            else:
 941                args = []
 942                kwargs = []
 943                dyn_args = dyn_kwargs = None
 944            node = nodes.Filter(
 945                node, name, args, kwargs, dyn_args, dyn_kwargs, lineno=token.lineno
 946            )
 947            start_inline = False
 948        return node
 949
 950    def parse_test(self, node: nodes.Expr) -> nodes.Expr:
 951        token = next(self.stream)
 952        if self.stream.current.test("name:not"):
 953            next(self.stream)
 954            negated = True
 955        else:
 956            negated = False
 957        name = self.stream.expect("name").value
 958        while self.stream.current.type == "dot":
 959            next(self.stream)
 960            name += "." + self.stream.expect("name").value
 961        dyn_args = dyn_kwargs = None
 962        kwargs: t.List[nodes.Keyword] = []
 963        if self.stream.current.type == "lparen":
 964            args, kwargs, dyn_args, dyn_kwargs = self.parse_call_args()
 965        elif self.stream.current.type in {
 966            "name",
 967            "string",
 968            "integer",
 969            "float",
 970            "lparen",
 971            "lbracket",
 972            "lbrace",
 973        } and not self.stream.current.test_any("name:else", "name:or", "name:and"):
 974            if self.stream.current.test("name:is"):
 975                self.fail("You cannot chain multiple tests with is")
 976            arg_node = self.parse_primary()
 977            arg_node = self.parse_postfix(arg_node)
 978            args = [arg_node]
 979        else:
 980            args = []
 981        node = nodes.Test(
 982            node, name, args, kwargs, dyn_args, dyn_kwargs, lineno=token.lineno
 983        )
 984        if negated:
 985            node = nodes.Not(node, lineno=token.lineno)
 986        return node
 987
 988    def subparse(
 989        self, end_tokens: t.Optional[t.Tuple[str, ...]] = None
 990    ) -> t.List[nodes.Node]:
 991        body: t.List[nodes.Node] = []
 992        data_buffer: t.List[nodes.Node] = []
 993        add_data = data_buffer.append
 994
 995        if end_tokens is not None:
 996            self._end_token_stack.append(end_tokens)
 997
 998        def flush_data() -> None:
 999            if data_buffer:
1000                lineno = data_buffer[0].lineno
1001                body.append(nodes.Output(data_buffer[:], lineno=lineno))
1002                del data_buffer[:]
1003
1004        try:
1005            while self.stream:
1006                token = self.stream.current
1007                if token.type == "data":
1008                    if token.value:
1009                        add_data(nodes.TemplateData(token.value, lineno=token.lineno))
1010                    next(self.stream)
1011                elif token.type == "variable_begin":
1012                    next(self.stream)
1013                    add_data(self.parse_tuple(with_condexpr=True))
1014                    self.stream.expect("variable_end")
1015                elif token.type == "block_begin":
1016                    flush_data()
1017                    next(self.stream)
1018                    if end_tokens is not None and self.stream.current.test_any(
1019                        *end_tokens
1020                    ):
1021                        return body
1022                    rv = self.parse_statement()
1023                    if isinstance(rv, list):
1024                        body.extend(rv)
1025                    else:
1026                        body.append(rv)
1027                    self.stream.expect("block_end")
1028                else:
1029                    raise AssertionError("internal parsing error")
1030
1031            flush_data()
1032        finally:
1033            if end_tokens is not None:
1034                self._end_token_stack.pop()
1035        return body
1036
1037    def parse(self) -> nodes.Template:
1038        """Parse the whole template into a `Template` node."""
1039        result = nodes.Template(self.subparse(), lineno=1)
1040        result.set_environment(self.environment)
1041        return result