jinja2.compiler

   1"""Compiles nodes from the parser into Python code."""
   2
   3import typing as t
   4from contextlib import contextmanager
   5from functools import update_wrapper
   6from io import StringIO
   7from itertools import chain
   8from keyword import iskeyword as is_python_keyword
   9
  10from markupsafe import escape
  11from markupsafe import Markup
  12
  13from . import nodes
  14from .exceptions import TemplateAssertionError
  15from .idtracking import Symbols
  16from .idtracking import VAR_LOAD_ALIAS
  17from .idtracking import VAR_LOAD_PARAMETER
  18from .idtracking import VAR_LOAD_RESOLVE
  19from .idtracking import VAR_LOAD_UNDEFINED
  20from .nodes import EvalContext
  21from .optimizer import Optimizer
  22from .utils import _PassArg
  23from .utils import concat
  24from .visitor import NodeVisitor
  25
  26if t.TYPE_CHECKING:
  27    import typing_extensions as te
  28
  29    from .environment import Environment
  30
  31F = t.TypeVar("F", bound=t.Callable[..., t.Any])
  32
  33operators = {
  34    "eq": "==",
  35    "ne": "!=",
  36    "gt": ">",
  37    "gteq": ">=",
  38    "lt": "<",
  39    "lteq": "<=",
  40    "in": "in",
  41    "notin": "not in",
  42}
  43
  44
  45def optimizeconst(f: F) -> F:
  46    def new_func(
  47        self: "CodeGenerator", node: nodes.Expr, frame: "Frame", **kwargs: t.Any
  48    ) -> t.Any:
  49        # Only optimize if the frame is not volatile
  50        if self.optimizer is not None and not frame.eval_ctx.volatile:
  51            new_node = self.optimizer.visit(node, frame.eval_ctx)
  52
  53            if new_node != node:
  54                return self.visit(new_node, frame)
  55
  56        return f(self, node, frame, **kwargs)
  57
  58    return update_wrapper(t.cast(F, new_func), f)
  59
  60
  61def _make_binop(op: str) -> t.Callable[["CodeGenerator", nodes.BinExpr, "Frame"], None]:
  62    @optimizeconst
  63    def visitor(self: "CodeGenerator", node: nodes.BinExpr, frame: Frame) -> None:
  64        if (
  65            self.environment.sandboxed and op in self.environment.intercepted_binops  # type: ignore
  66        ):
  67            self.write(f"environment.call_binop(context, {op!r}, ")
  68            self.visit(node.left, frame)
  69            self.write(", ")
  70            self.visit(node.right, frame)
  71        else:
  72            self.write("(")
  73            self.visit(node.left, frame)
  74            self.write(f" {op} ")
  75            self.visit(node.right, frame)
  76
  77        self.write(")")
  78
  79    return visitor
  80
  81
  82def _make_unop(
  83    op: str,
  84) -> t.Callable[["CodeGenerator", nodes.UnaryExpr, "Frame"], None]:
  85    @optimizeconst
  86    def visitor(self: "CodeGenerator", node: nodes.UnaryExpr, frame: Frame) -> None:
  87        if (
  88            self.environment.sandboxed and op in self.environment.intercepted_unops  # type: ignore
  89        ):
  90            self.write(f"environment.call_unop(context, {op!r}, ")
  91            self.visit(node.node, frame)
  92        else:
  93            self.write("(" + op)
  94            self.visit(node.node, frame)
  95
  96        self.write(")")
  97
  98    return visitor
  99
 100
 101def generate(
 102    node: nodes.Template,
 103    environment: "Environment",
 104    name: t.Optional[str],
 105    filename: t.Optional[str],
 106    stream: t.Optional[t.TextIO] = None,
 107    defer_init: bool = False,
 108    optimized: bool = True,
 109) -> t.Optional[str]:
 110    """Generate the python source for a node tree."""
 111    if not isinstance(node, nodes.Template):
 112        raise TypeError("Can't compile non template nodes")
 113
 114    generator = environment.code_generator_class(
 115        environment, name, filename, stream, defer_init, optimized
 116    )
 117    generator.visit(node)
 118
 119    if stream is None:
 120        return generator.stream.getvalue()  # type: ignore
 121
 122    return None
 123
 124
 125def has_safe_repr(value: t.Any) -> bool:
 126    """Does the node have a safe representation?"""
 127    if value is None or value is NotImplemented or value is Ellipsis:
 128        return True
 129
 130    if type(value) in {bool, int, float, complex, range, str, Markup}:
 131        return True
 132
 133    if type(value) in {tuple, list, set, frozenset}:
 134        return all(has_safe_repr(v) for v in value)
 135
 136    if type(value) is dict:  # noqa E721
 137        return all(has_safe_repr(k) and has_safe_repr(v) for k, v in value.items())
 138
 139    return False
 140
 141
 142def find_undeclared(
 143    nodes: t.Iterable[nodes.Node], names: t.Iterable[str]
 144) -> t.Set[str]:
 145    """Check if the names passed are accessed undeclared.  The return value
 146    is a set of all the undeclared names from the sequence of names found.
 147    """
 148    visitor = UndeclaredNameVisitor(names)
 149    try:
 150        for node in nodes:
 151            visitor.visit(node)
 152    except VisitorExit:
 153        pass
 154    return visitor.undeclared
 155
 156
 157class MacroRef:
 158    def __init__(self, node: t.Union[nodes.Macro, nodes.CallBlock]) -> None:
 159        self.node = node
 160        self.accesses_caller = False
 161        self.accesses_kwargs = False
 162        self.accesses_varargs = False
 163
 164
 165class Frame:
 166    """Holds compile time information for us."""
 167
 168    def __init__(
 169        self,
 170        eval_ctx: EvalContext,
 171        parent: t.Optional["Frame"] = None,
 172        level: t.Optional[int] = None,
 173    ) -> None:
 174        self.eval_ctx = eval_ctx
 175
 176        # the parent of this frame
 177        self.parent = parent
 178
 179        if parent is None:
 180            self.symbols = Symbols(level=level)
 181
 182            # in some dynamic inheritance situations the compiler needs to add
 183            # write tests around output statements.
 184            self.require_output_check = False
 185
 186            # inside some tags we are using a buffer rather than yield statements.
 187            # this for example affects {% filter %} or {% macro %}.  If a frame
 188            # is buffered this variable points to the name of the list used as
 189            # buffer.
 190            self.buffer: t.Optional[str] = None
 191
 192            # the name of the block we're in, otherwise None.
 193            self.block: t.Optional[str] = None
 194
 195        else:
 196            self.symbols = Symbols(parent.symbols, level=level)
 197            self.require_output_check = parent.require_output_check
 198            self.buffer = parent.buffer
 199            self.block = parent.block
 200
 201        # a toplevel frame is the root + soft frames such as if conditions.
 202        self.toplevel = False
 203
 204        # the root frame is basically just the outermost frame, so no if
 205        # conditions.  This information is used to optimize inheritance
 206        # situations.
 207        self.rootlevel = False
 208
 209        # variables set inside of loops and blocks should not affect outer frames,
 210        # but they still needs to be kept track of as part of the active context.
 211        self.loop_frame = False
 212        self.block_frame = False
 213
 214        # track whether the frame is being used in an if-statement or conditional
 215        # expression as it determines which errors should be raised during runtime
 216        # or compile time.
 217        self.soft_frame = False
 218
 219    def copy(self) -> "Frame":
 220        """Create a copy of the current one."""
 221        rv = object.__new__(self.__class__)
 222        rv.__dict__.update(self.__dict__)
 223        rv.symbols = self.symbols.copy()
 224        return rv
 225
 226    def inner(self, isolated: bool = False) -> "Frame":
 227        """Return an inner frame."""
 228        if isolated:
 229            return Frame(self.eval_ctx, level=self.symbols.level + 1)
 230        return Frame(self.eval_ctx, self)
 231
 232    def soft(self) -> "Frame":
 233        """Return a soft frame.  A soft frame may not be modified as
 234        standalone thing as it shares the resources with the frame it
 235        was created of, but it's not a rootlevel frame any longer.
 236
 237        This is only used to implement if-statements and conditional
 238        expressions.
 239        """
 240        rv = self.copy()
 241        rv.rootlevel = False
 242        rv.soft_frame = True
 243        return rv
 244
 245    __copy__ = copy
 246
 247
 248class VisitorExit(RuntimeError):
 249    """Exception used by the `UndeclaredNameVisitor` to signal a stop."""
 250
 251
 252class DependencyFinderVisitor(NodeVisitor):
 253    """A visitor that collects filter and test calls."""
 254
 255    def __init__(self) -> None:
 256        self.filters: t.Set[str] = set()
 257        self.tests: t.Set[str] = set()
 258
 259    def visit_Filter(self, node: nodes.Filter) -> None:
 260        self.generic_visit(node)
 261        self.filters.add(node.name)
 262
 263    def visit_Test(self, node: nodes.Test) -> None:
 264        self.generic_visit(node)
 265        self.tests.add(node.name)
 266
 267    def visit_Block(self, node: nodes.Block) -> None:
 268        """Stop visiting at blocks."""
 269
 270
 271class UndeclaredNameVisitor(NodeVisitor):
 272    """A visitor that checks if a name is accessed without being
 273    declared.  This is different from the frame visitor as it will
 274    not stop at closure frames.
 275    """
 276
 277    def __init__(self, names: t.Iterable[str]) -> None:
 278        self.names = set(names)
 279        self.undeclared: t.Set[str] = set()
 280
 281    def visit_Name(self, node: nodes.Name) -> None:
 282        if node.ctx == "load" and node.name in self.names:
 283            self.undeclared.add(node.name)
 284            if self.undeclared == self.names:
 285                raise VisitorExit()
 286        else:
 287            self.names.discard(node.name)
 288
 289    def visit_Block(self, node: nodes.Block) -> None:
 290        """Stop visiting a blocks."""
 291
 292
 293class CompilerExit(Exception):
 294    """Raised if the compiler encountered a situation where it just
 295    doesn't make sense to further process the code.  Any block that
 296    raises such an exception is not further processed.
 297    """
 298
 299
 300class CodeGenerator(NodeVisitor):
 301    def __init__(
 302        self,
 303        environment: "Environment",
 304        name: t.Optional[str],
 305        filename: t.Optional[str],
 306        stream: t.Optional[t.TextIO] = None,
 307        defer_init: bool = False,
 308        optimized: bool = True,
 309    ) -> None:
 310        if stream is None:
 311            stream = StringIO()
 312        self.environment = environment
 313        self.name = name
 314        self.filename = filename
 315        self.stream = stream
 316        self.created_block_context = False
 317        self.defer_init = defer_init
 318        self.optimizer: t.Optional[Optimizer] = None
 319
 320        if optimized:
 321            self.optimizer = Optimizer(environment)
 322
 323        # aliases for imports
 324        self.import_aliases: t.Dict[str, str] = {}
 325
 326        # a registry for all blocks.  Because blocks are moved out
 327        # into the global python scope they are registered here
 328        self.blocks: t.Dict[str, nodes.Block] = {}
 329
 330        # the number of extends statements so far
 331        self.extends_so_far = 0
 332
 333        # some templates have a rootlevel extends.  In this case we
 334        # can safely assume that we're a child template and do some
 335        # more optimizations.
 336        self.has_known_extends = False
 337
 338        # the current line number
 339        self.code_lineno = 1
 340
 341        # registry of all filters and tests (global, not block local)
 342        self.tests: t.Dict[str, str] = {}
 343        self.filters: t.Dict[str, str] = {}
 344
 345        # the debug information
 346        self.debug_info: t.List[t.Tuple[int, int]] = []
 347        self._write_debug_info: t.Optional[int] = None
 348
 349        # the number of new lines before the next write()
 350        self._new_lines = 0
 351
 352        # the line number of the last written statement
 353        self._last_line = 0
 354
 355        # true if nothing was written so far.
 356        self._first_write = True
 357
 358        # used by the `temporary_identifier` method to get new
 359        # unique, temporary identifier
 360        self._last_identifier = 0
 361
 362        # the current indentation
 363        self._indentation = 0
 364
 365        # Tracks toplevel assignments
 366        self._assign_stack: t.List[t.Set[str]] = []
 367
 368        # Tracks parameter definition blocks
 369        self._param_def_block: t.List[t.Set[str]] = []
 370
 371        # Tracks the current context.
 372        self._context_reference_stack = ["context"]
 373
 374    @property
 375    def optimized(self) -> bool:
 376        return self.optimizer is not None
 377
 378    # -- Various compilation helpers
 379
 380    def fail(self, msg: str, lineno: int) -> "te.NoReturn":
 381        """Fail with a :exc:`TemplateAssertionError`."""
 382        raise TemplateAssertionError(msg, lineno, self.name, self.filename)
 383
 384    def temporary_identifier(self) -> str:
 385        """Get a new unique identifier."""
 386        self._last_identifier += 1
 387        return f"t_{self._last_identifier}"
 388
 389    def buffer(self, frame: Frame) -> None:
 390        """Enable buffering for the frame from that point onwards."""
 391        frame.buffer = self.temporary_identifier()
 392        self.writeline(f"{frame.buffer} = []")
 393
 394    def return_buffer_contents(
 395        self, frame: Frame, force_unescaped: bool = False
 396    ) -> None:
 397        """Return the buffer contents of the frame."""
 398        if not force_unescaped:
 399            if frame.eval_ctx.volatile:
 400                self.writeline("if context.eval_ctx.autoescape:")
 401                self.indent()
 402                self.writeline(f"return Markup(concat({frame.buffer}))")
 403                self.outdent()
 404                self.writeline("else:")
 405                self.indent()
 406                self.writeline(f"return concat({frame.buffer})")
 407                self.outdent()
 408                return
 409            elif frame.eval_ctx.autoescape:
 410                self.writeline(f"return Markup(concat({frame.buffer}))")
 411                return
 412        self.writeline(f"return concat({frame.buffer})")
 413
 414    def indent(self) -> None:
 415        """Indent by one."""
 416        self._indentation += 1
 417
 418    def outdent(self, step: int = 1) -> None:
 419        """Outdent by step."""
 420        self._indentation -= step
 421
 422    def start_write(self, frame: Frame, node: t.Optional[nodes.Node] = None) -> None:
 423        """Yield or write into the frame buffer."""
 424        if frame.buffer is None:
 425            self.writeline("yield ", node)
 426        else:
 427            self.writeline(f"{frame.buffer}.append(", node)
 428
 429    def end_write(self, frame: Frame) -> None:
 430        """End the writing process started by `start_write`."""
 431        if frame.buffer is not None:
 432            self.write(")")
 433
 434    def simple_write(
 435        self, s: str, frame: Frame, node: t.Optional[nodes.Node] = None
 436    ) -> None:
 437        """Simple shortcut for start_write + write + end_write."""
 438        self.start_write(frame, node)
 439        self.write(s)
 440        self.end_write(frame)
 441
 442    def blockvisit(self, nodes: t.Iterable[nodes.Node], frame: Frame) -> None:
 443        """Visit a list of nodes as block in a frame.  If the current frame
 444        is no buffer a dummy ``if 0: yield None`` is written automatically.
 445        """
 446        try:
 447            self.writeline("pass")
 448            for node in nodes:
 449                self.visit(node, frame)
 450        except CompilerExit:
 451            pass
 452
 453    def write(self, x: str) -> None:
 454        """Write a string into the output stream."""
 455        if self._new_lines:
 456            if not self._first_write:
 457                self.stream.write("\n" * self._new_lines)
 458                self.code_lineno += self._new_lines
 459                if self._write_debug_info is not None:
 460                    self.debug_info.append((self._write_debug_info, self.code_lineno))
 461                    self._write_debug_info = None
 462            self._first_write = False
 463            self.stream.write("    " * self._indentation)
 464            self._new_lines = 0
 465        self.stream.write(x)
 466
 467    def writeline(
 468        self, x: str, node: t.Optional[nodes.Node] = None, extra: int = 0
 469    ) -> None:
 470        """Combination of newline and write."""
 471        self.newline(node, extra)
 472        self.write(x)
 473
 474    def newline(self, node: t.Optional[nodes.Node] = None, extra: int = 0) -> None:
 475        """Add one or more newlines before the next write."""
 476        self._new_lines = max(self._new_lines, 1 + extra)
 477        if node is not None and node.lineno != self._last_line:
 478            self._write_debug_info = node.lineno
 479            self._last_line = node.lineno
 480
 481    def signature(
 482        self,
 483        node: t.Union[nodes.Call, nodes.Filter, nodes.Test],
 484        frame: Frame,
 485        extra_kwargs: t.Optional[t.Mapping[str, t.Any]] = None,
 486    ) -> None:
 487        """Writes a function call to the stream for the current node.
 488        A leading comma is added automatically.  The extra keyword
 489        arguments may not include python keywords otherwise a syntax
 490        error could occur.  The extra keyword arguments should be given
 491        as python dict.
 492        """
 493        # if any of the given keyword arguments is a python keyword
 494        # we have to make sure that no invalid call is created.
 495        kwarg_workaround = any(
 496            is_python_keyword(t.cast(str, k))
 497            for k in chain((x.key for x in node.kwargs), extra_kwargs or ())
 498        )
 499
 500        for arg in node.args:
 501            self.write(", ")
 502            self.visit(arg, frame)
 503
 504        if not kwarg_workaround:
 505            for kwarg in node.kwargs:
 506                self.write(", ")
 507                self.visit(kwarg, frame)
 508            if extra_kwargs is not None:
 509                for key, value in extra_kwargs.items():
 510                    self.write(f", {key}={value}")
 511        if node.dyn_args:
 512            self.write(", *")
 513            self.visit(node.dyn_args, frame)
 514
 515        if kwarg_workaround:
 516            if node.dyn_kwargs is not None:
 517                self.write(", **dict({")
 518            else:
 519                self.write(", **{")
 520            for kwarg in node.kwargs:
 521                self.write(f"{kwarg.key!r}: ")
 522                self.visit(kwarg.value, frame)
 523                self.write(", ")
 524            if extra_kwargs is not None:
 525                for key, value in extra_kwargs.items():
 526                    self.write(f"{key!r}: {value}, ")
 527            if node.dyn_kwargs is not None:
 528                self.write("}, **")
 529                self.visit(node.dyn_kwargs, frame)
 530                self.write(")")
 531            else:
 532                self.write("}")
 533
 534        elif node.dyn_kwargs is not None:
 535            self.write(", **")
 536            self.visit(node.dyn_kwargs, frame)
 537
 538    def pull_dependencies(self, nodes: t.Iterable[nodes.Node]) -> None:
 539        """Find all filter and test names used in the template and
 540        assign them to variables in the compiled namespace. Checking
 541        that the names are registered with the environment is done when
 542        compiling the Filter and Test nodes. If the node is in an If or
 543        CondExpr node, the check is done at runtime instead.
 544
 545        .. versionchanged:: 3.0
 546            Filters and tests in If and CondExpr nodes are checked at
 547            runtime instead of compile time.
 548        """
 549        visitor = DependencyFinderVisitor()
 550
 551        for node in nodes:
 552            visitor.visit(node)
 553
 554        for id_map, names, dependency in (
 555            (self.filters, visitor.filters, "filters"),
 556            (
 557                self.tests,
 558                visitor.tests,
 559                "tests",
 560            ),
 561        ):
 562            for name in sorted(names):
 563                if name not in id_map:
 564                    id_map[name] = self.temporary_identifier()
 565
 566                # add check during runtime that dependencies used inside of executed
 567                # blocks are defined, as this step may be skipped during compile time
 568                self.writeline("try:")
 569                self.indent()
 570                self.writeline(f"{id_map[name]} = environment.{dependency}[{name!r}]")
 571                self.outdent()
 572                self.writeline("except KeyError:")
 573                self.indent()
 574                self.writeline("@internalcode")
 575                self.writeline(f"def {id_map[name]}(*unused):")
 576                self.indent()
 577                self.writeline(
 578                    f'raise TemplateRuntimeError("No {dependency[:-1]}'
 579                    f' named {name!r} found.")'
 580                )
 581                self.outdent()
 582                self.outdent()
 583
 584    def enter_frame(self, frame: Frame) -> None:
 585        undefs = []
 586        for target, (action, param) in frame.symbols.loads.items():
 587            if action == VAR_LOAD_PARAMETER:
 588                pass
 589            elif action == VAR_LOAD_RESOLVE:
 590                self.writeline(f"{target} = {self.get_resolve_func()}({param!r})")
 591            elif action == VAR_LOAD_ALIAS:
 592                self.writeline(f"{target} = {param}")
 593            elif action == VAR_LOAD_UNDEFINED:
 594                undefs.append(target)
 595            else:
 596                raise NotImplementedError("unknown load instruction")
 597        if undefs:
 598            self.writeline(f"{' = '.join(undefs)} = missing")
 599
 600    def leave_frame(self, frame: Frame, with_python_scope: bool = False) -> None:
 601        if not with_python_scope:
 602            undefs = []
 603            for target in frame.symbols.loads:
 604                undefs.append(target)
 605            if undefs:
 606                self.writeline(f"{' = '.join(undefs)} = missing")
 607
 608    def choose_async(self, async_value: str = "async ", sync_value: str = "") -> str:
 609        return async_value if self.environment.is_async else sync_value
 610
 611    def func(self, name: str) -> str:
 612        return f"{self.choose_async()}def {name}"
 613
 614    def macro_body(
 615        self, node: t.Union[nodes.Macro, nodes.CallBlock], frame: Frame
 616    ) -> t.Tuple[Frame, MacroRef]:
 617        """Dump the function def of a macro or call block."""
 618        frame = frame.inner()
 619        frame.symbols.analyze_node(node)
 620        macro_ref = MacroRef(node)
 621
 622        explicit_caller = None
 623        skip_special_params = set()
 624        args = []
 625
 626        for idx, arg in enumerate(node.args):
 627            if arg.name == "caller":
 628                explicit_caller = idx
 629            if arg.name in ("kwargs", "varargs"):
 630                skip_special_params.add(arg.name)
 631            args.append(frame.symbols.ref(arg.name))
 632
 633        undeclared = find_undeclared(node.body, ("caller", "kwargs", "varargs"))
 634
 635        if "caller" in undeclared:
 636            # In older Jinja versions there was a bug that allowed caller
 637            # to retain the special behavior even if it was mentioned in
 638            # the argument list.  However thankfully this was only really
 639            # working if it was the last argument.  So we are explicitly
 640            # checking this now and error out if it is anywhere else in
 641            # the argument list.
 642            if explicit_caller is not None:
 643                try:
 644                    node.defaults[explicit_caller - len(node.args)]
 645                except IndexError:
 646                    self.fail(
 647                        "When defining macros or call blocks the "
 648                        'special "caller" argument must be omitted '
 649                        "or be given a default.",
 650                        node.lineno,
 651                    )
 652            else:
 653                args.append(frame.symbols.declare_parameter("caller"))
 654            macro_ref.accesses_caller = True
 655        if "kwargs" in undeclared and "kwargs" not in skip_special_params:
 656            args.append(frame.symbols.declare_parameter("kwargs"))
 657            macro_ref.accesses_kwargs = True
 658        if "varargs" in undeclared and "varargs" not in skip_special_params:
 659            args.append(frame.symbols.declare_parameter("varargs"))
 660            macro_ref.accesses_varargs = True
 661
 662        # macros are delayed, they never require output checks
 663        frame.require_output_check = False
 664        frame.symbols.analyze_node(node)
 665        self.writeline(f"{self.func('macro')}({', '.join(args)}):", node)
 666        self.indent()
 667
 668        self.buffer(frame)
 669        self.enter_frame(frame)
 670
 671        self.push_parameter_definitions(frame)
 672        for idx, arg in enumerate(node.args):
 673            ref = frame.symbols.ref(arg.name)
 674            self.writeline(f"if {ref} is missing:")
 675            self.indent()
 676            try:
 677                default = node.defaults[idx - len(node.args)]
 678            except IndexError:
 679                self.writeline(
 680                    f'{ref} = undefined("parameter {arg.name!r} was not provided",'
 681                    f" name={arg.name!r})"
 682                )
 683            else:
 684                self.writeline(f"{ref} = ")
 685                self.visit(default, frame)
 686            self.mark_parameter_stored(ref)
 687            self.outdent()
 688        self.pop_parameter_definitions()
 689
 690        self.blockvisit(node.body, frame)
 691        self.return_buffer_contents(frame, force_unescaped=True)
 692        self.leave_frame(frame, with_python_scope=True)
 693        self.outdent()
 694
 695        return frame, macro_ref
 696
 697    def macro_def(self, macro_ref: MacroRef, frame: Frame) -> None:
 698        """Dump the macro definition for the def created by macro_body."""
 699        arg_tuple = ", ".join(repr(x.name) for x in macro_ref.node.args)
 700        name = getattr(macro_ref.node, "name", None)
 701        if len(macro_ref.node.args) == 1:
 702            arg_tuple += ","
 703        self.write(
 704            f"Macro(environment, macro, {name!r}, ({arg_tuple}),"
 705            f" {macro_ref.accesses_kwargs!r}, {macro_ref.accesses_varargs!r},"
 706            f" {macro_ref.accesses_caller!r}, context.eval_ctx.autoescape)"
 707        )
 708
 709    def position(self, node: nodes.Node) -> str:
 710        """Return a human readable position for the node."""
 711        rv = f"line {node.lineno}"
 712        if self.name is not None:
 713            rv = f"{rv} in {self.name!r}"
 714        return rv
 715
 716    def dump_local_context(self, frame: Frame) -> str:
 717        items_kv = ", ".join(
 718            f"{name!r}: {target}"
 719            for name, target in frame.symbols.dump_stores().items()
 720        )
 721        return f"{{{items_kv}}}"
 722
 723    def write_commons(self) -> None:
 724        """Writes a common preamble that is used by root and block functions.
 725        Primarily this sets up common local helpers and enforces a generator
 726        through a dead branch.
 727        """
 728        self.writeline("resolve = context.resolve_or_missing")
 729        self.writeline("undefined = environment.undefined")
 730        self.writeline("concat = environment.concat")
 731        # always use the standard Undefined class for the implicit else of
 732        # conditional expressions
 733        self.writeline("cond_expr_undefined = Undefined")
 734        self.writeline("if 0: yield None")
 735
 736    def push_parameter_definitions(self, frame: Frame) -> None:
 737        """Pushes all parameter targets from the given frame into a local
 738        stack that permits tracking of yet to be assigned parameters.  In
 739        particular this enables the optimization from `visit_Name` to skip
 740        undefined expressions for parameters in macros as macros can reference
 741        otherwise unbound parameters.
 742        """
 743        self._param_def_block.append(frame.symbols.dump_param_targets())
 744
 745    def pop_parameter_definitions(self) -> None:
 746        """Pops the current parameter definitions set."""
 747        self._param_def_block.pop()
 748
 749    def mark_parameter_stored(self, target: str) -> None:
 750        """Marks a parameter in the current parameter definitions as stored.
 751        This will skip the enforced undefined checks.
 752        """
 753        if self._param_def_block:
 754            self._param_def_block[-1].discard(target)
 755
 756    def push_context_reference(self, target: str) -> None:
 757        self._context_reference_stack.append(target)
 758
 759    def pop_context_reference(self) -> None:
 760        self._context_reference_stack.pop()
 761
 762    def get_context_ref(self) -> str:
 763        return self._context_reference_stack[-1]
 764
 765    def get_resolve_func(self) -> str:
 766        target = self._context_reference_stack[-1]
 767        if target == "context":
 768            return "resolve"
 769        return f"{target}.resolve"
 770
 771    def derive_context(self, frame: Frame) -> str:
 772        return f"{self.get_context_ref()}.derived({self.dump_local_context(frame)})"
 773
 774    def parameter_is_undeclared(self, target: str) -> bool:
 775        """Checks if a given target is an undeclared parameter."""
 776        if not self._param_def_block:
 777            return False
 778        return target in self._param_def_block[-1]
 779
 780    def push_assign_tracking(self) -> None:
 781        """Pushes a new layer for assignment tracking."""
 782        self._assign_stack.append(set())
 783
 784    def pop_assign_tracking(self, frame: Frame) -> None:
 785        """Pops the topmost level for assignment tracking and updates the
 786        context variables if necessary.
 787        """
 788        vars = self._assign_stack.pop()
 789        if (
 790            not frame.block_frame
 791            and not frame.loop_frame
 792            and not frame.toplevel
 793            or not vars
 794        ):
 795            return
 796        public_names = [x for x in vars if x[:1] != "_"]
 797        if len(vars) == 1:
 798            name = next(iter(vars))
 799            ref = frame.symbols.ref(name)
 800            if frame.loop_frame:
 801                self.writeline(f"_loop_vars[{name!r}] = {ref}")
 802                return
 803            if frame.block_frame:
 804                self.writeline(f"_block_vars[{name!r}] = {ref}")
 805                return
 806            self.writeline(f"context.vars[{name!r}] = {ref}")
 807        else:
 808            if frame.loop_frame:
 809                self.writeline("_loop_vars.update({")
 810            elif frame.block_frame:
 811                self.writeline("_block_vars.update({")
 812            else:
 813                self.writeline("context.vars.update({")
 814            for idx, name in enumerate(vars):
 815                if idx:
 816                    self.write(", ")
 817                ref = frame.symbols.ref(name)
 818                self.write(f"{name!r}: {ref}")
 819            self.write("})")
 820        if not frame.block_frame and not frame.loop_frame and public_names:
 821            if len(public_names) == 1:
 822                self.writeline(f"context.exported_vars.add({public_names[0]!r})")
 823            else:
 824                names_str = ", ".join(map(repr, public_names))
 825                self.writeline(f"context.exported_vars.update(({names_str}))")
 826
 827    # -- Statement Visitors
 828
 829    def visit_Template(
 830        self, node: nodes.Template, frame: t.Optional[Frame] = None
 831    ) -> None:
 832        assert frame is None, "no root frame allowed"
 833        eval_ctx = EvalContext(self.environment, self.name)
 834
 835        from .runtime import async_exported
 836        from .runtime import exported
 837
 838        if self.environment.is_async:
 839            exported_names = sorted(exported + async_exported)
 840        else:
 841            exported_names = sorted(exported)
 842
 843        self.writeline("from jinja2.runtime import " + ", ".join(exported_names))
 844
 845        # if we want a deferred initialization we cannot move the
 846        # environment into a local name
 847        envenv = "" if self.defer_init else ", environment=environment"
 848
 849        # do we have an extends tag at all?  If not, we can save some
 850        # overhead by just not processing any inheritance code.
 851        have_extends = node.find(nodes.Extends) is not None
 852
 853        # find all blocks
 854        for block in node.find_all(nodes.Block):
 855            if block.name in self.blocks:
 856                self.fail(f"block {block.name!r} defined twice", block.lineno)
 857            self.blocks[block.name] = block
 858
 859        # find all imports and import them
 860        for import_ in node.find_all(nodes.ImportedName):
 861            if import_.importname not in self.import_aliases:
 862                imp = import_.importname
 863                self.import_aliases[imp] = alias = self.temporary_identifier()
 864                if "." in imp:
 865                    module, obj = imp.rsplit(".", 1)
 866                    self.writeline(f"from {module} import {obj} as {alias}")
 867                else:
 868                    self.writeline(f"import {imp} as {alias}")
 869
 870        # add the load name
 871        self.writeline(f"name = {self.name!r}")
 872
 873        # generate the root render function.
 874        self.writeline(
 875            f"{self.func('root')}(context, missing=missing{envenv}):", extra=1
 876        )
 877        self.indent()
 878        self.write_commons()
 879
 880        # process the root
 881        frame = Frame(eval_ctx)
 882        if "self" in find_undeclared(node.body, ("self",)):
 883            ref = frame.symbols.declare_parameter("self")
 884            self.writeline(f"{ref} = TemplateReference(context)")
 885        frame.symbols.analyze_node(node)
 886        frame.toplevel = frame.rootlevel = True
 887        frame.require_output_check = have_extends and not self.has_known_extends
 888        if have_extends:
 889            self.writeline("parent_template = None")
 890        self.enter_frame(frame)
 891        self.pull_dependencies(node.body)
 892        self.blockvisit(node.body, frame)
 893        self.leave_frame(frame, with_python_scope=True)
 894        self.outdent()
 895
 896        # make sure that the parent root is called.
 897        if have_extends:
 898            if not self.has_known_extends:
 899                self.indent()
 900                self.writeline("if parent_template is not None:")
 901            self.indent()
 902            if not self.environment.is_async:
 903                self.writeline("yield from parent_template.root_render_func(context)")
 904            else:
 905                self.writeline(
 906                    "async for event in parent_template.root_render_func(context):"
 907                )
 908                self.indent()
 909                self.writeline("yield event")
 910                self.outdent()
 911            self.outdent(1 + (not self.has_known_extends))
 912
 913        # at this point we now have the blocks collected and can visit them too.
 914        for name, block in self.blocks.items():
 915            self.writeline(
 916                f"{self.func('block_' + name)}(context, missing=missing{envenv}):",
 917                block,
 918                1,
 919            )
 920            self.indent()
 921            self.write_commons()
 922            # It's important that we do not make this frame a child of the
 923            # toplevel template.  This would cause a variety of
 924            # interesting issues with identifier tracking.
 925            block_frame = Frame(eval_ctx)
 926            block_frame.block_frame = True
 927            undeclared = find_undeclared(block.body, ("self", "super"))
 928            if "self" in undeclared:
 929                ref = block_frame.symbols.declare_parameter("self")
 930                self.writeline(f"{ref} = TemplateReference(context)")
 931            if "super" in undeclared:
 932                ref = block_frame.symbols.declare_parameter("super")
 933                self.writeline(f"{ref} = context.super({name!r}, block_{name})")
 934            block_frame.symbols.analyze_node(block)
 935            block_frame.block = name
 936            self.writeline("_block_vars = {}")
 937            self.enter_frame(block_frame)
 938            self.pull_dependencies(block.body)
 939            self.blockvisit(block.body, block_frame)
 940            self.leave_frame(block_frame, with_python_scope=True)
 941            self.outdent()
 942
 943        blocks_kv_str = ", ".join(f"{x!r}: block_{x}" for x in self.blocks)
 944        self.writeline(f"blocks = {{{blocks_kv_str}}}", extra=1)
 945        debug_kv_str = "&".join(f"{k}={v}" for k, v in self.debug_info)
 946        self.writeline(f"debug_info = {debug_kv_str!r}")
 947
 948    def visit_Block(self, node: nodes.Block, frame: Frame) -> None:
 949        """Call a block and register it for the template."""
 950        level = 0
 951        if frame.toplevel:
 952            # if we know that we are a child template, there is no need to
 953            # check if we are one
 954            if self.has_known_extends:
 955                return
 956            if self.extends_so_far > 0:
 957                self.writeline("if parent_template is None:")
 958                self.indent()
 959                level += 1
 960
 961        if node.scoped:
 962            context = self.derive_context(frame)
 963        else:
 964            context = self.get_context_ref()
 965
 966        if node.required:
 967            self.writeline(f"if len(context.blocks[{node.name!r}]) <= 1:", node)
 968            self.indent()
 969            self.writeline(
 970                f'raise TemplateRuntimeError("Required block {node.name!r} not found")',
 971                node,
 972            )
 973            self.outdent()
 974
 975        if not self.environment.is_async and frame.buffer is None:
 976            self.writeline(
 977                f"yield from context.blocks[{node.name!r}][0]({context})", node
 978            )
 979        else:
 980            self.writeline(
 981                f"{self.choose_async()}for event in"
 982                f" context.blocks[{node.name!r}][0]({context}):",
 983                node,
 984            )
 985            self.indent()
 986            self.simple_write("event", frame)
 987            self.outdent()
 988
 989        self.outdent(level)
 990
 991    def visit_Extends(self, node: nodes.Extends, frame: Frame) -> None:
 992        """Calls the extender."""
 993        if not frame.toplevel:
 994            self.fail("cannot use extend from a non top-level scope", node.lineno)
 995
 996        # if the number of extends statements in general is zero so
 997        # far, we don't have to add a check if something extended
 998        # the template before this one.
 999        if self.extends_so_far > 0:
1000            # if we have a known extends we just add a template runtime
1001            # error into the generated code.  We could catch that at compile
1002            # time too, but i welcome it not to confuse users by throwing the
1003            # same error at different times just "because we can".
1004            if not self.has_known_extends:
1005                self.writeline("if parent_template is not None:")
1006                self.indent()
1007            self.writeline('raise TemplateRuntimeError("extended multiple times")')
1008
1009            # if we have a known extends already we don't need that code here
1010            # as we know that the template execution will end here.
1011            if self.has_known_extends:
1012                raise CompilerExit()
1013            else:
1014                self.outdent()
1015
1016        self.writeline("parent_template = environment.get_template(", node)
1017        self.visit(node.template, frame)
1018        self.write(f", {self.name!r})")
1019        self.writeline("for name, parent_block in parent_template.blocks.items():")
1020        self.indent()
1021        self.writeline("context.blocks.setdefault(name, []).append(parent_block)")
1022        self.outdent()
1023
1024        # if this extends statement was in the root level we can take
1025        # advantage of that information and simplify the generated code
1026        # in the top level from this point onwards
1027        if frame.rootlevel:
1028            self.has_known_extends = True
1029
1030        # and now we have one more
1031        self.extends_so_far += 1
1032
1033    def visit_Include(self, node: nodes.Include, frame: Frame) -> None:
1034        """Handles includes."""
1035        if node.ignore_missing:
1036            self.writeline("try:")
1037            self.indent()
1038
1039        func_name = "get_or_select_template"
1040        if isinstance(node.template, nodes.Const):
1041            if isinstance(node.template.value, str):
1042                func_name = "get_template"
1043            elif isinstance(node.template.value, (tuple, list)):
1044                func_name = "select_template"
1045        elif isinstance(node.template, (nodes.Tuple, nodes.List)):
1046            func_name = "select_template"
1047
1048        self.writeline(f"template = environment.{func_name}(", node)
1049        self.visit(node.template, frame)
1050        self.write(f", {self.name!r})")
1051        if node.ignore_missing:
1052            self.outdent()
1053            self.writeline("except TemplateNotFound:")
1054            self.indent()
1055            self.writeline("pass")
1056            self.outdent()
1057            self.writeline("else:")
1058            self.indent()
1059
1060        skip_event_yield = False
1061        if node.with_context:
1062            self.writeline(
1063                f"{self.choose_async()}for event in template.root_render_func("
1064                "template.new_context(context.get_all(), True,"
1065                f" {self.dump_local_context(frame)})):"
1066            )
1067        elif self.environment.is_async:
1068            self.writeline(
1069                "for event in (await template._get_default_module_async())"
1070                "._body_stream:"
1071            )
1072        else:
1073            self.writeline("yield from template._get_default_module()._body_stream")
1074            skip_event_yield = True
1075
1076        if not skip_event_yield:
1077            self.indent()
1078            self.simple_write("event", frame)
1079            self.outdent()
1080
1081        if node.ignore_missing:
1082            self.outdent()
1083
1084    def _import_common(
1085        self, node: t.Union[nodes.Import, nodes.FromImport], frame: Frame
1086    ) -> None:
1087        self.write(f"{self.choose_async('await ')}environment.get_template(")
1088        self.visit(node.template, frame)
1089        self.write(f", {self.name!r}).")
1090
1091        if node.with_context:
1092            f_name = f"make_module{self.choose_async('_async')}"
1093            self.write(
1094                f"{f_name}(context.get_all(), True, {self.dump_local_context(frame)})"
1095            )
1096        else:
1097            self.write(f"_get_default_module{self.choose_async('_async')}(context)")
1098
1099    def visit_Import(self, node: nodes.Import, frame: Frame) -> None:
1100        """Visit regular imports."""
1101        self.writeline(f"{frame.symbols.ref(node.target)} = ", node)
1102        if frame.toplevel:
1103            self.write(f"context.vars[{node.target!r}] = ")
1104
1105        self._import_common(node, frame)
1106
1107        if frame.toplevel and not node.target.startswith("_"):
1108            self.writeline(f"context.exported_vars.discard({node.target!r})")
1109
1110    def visit_FromImport(self, node: nodes.FromImport, frame: Frame) -> None:
1111        """Visit named imports."""
1112        self.newline(node)
1113        self.write("included_template = ")
1114        self._import_common(node, frame)
1115        var_names = []
1116        discarded_names = []
1117        for name in node.names:
1118            if isinstance(name, tuple):
1119                name, alias = name
1120            else:
1121                alias = name
1122            self.writeline(
1123                f"{frame.symbols.ref(alias)} ="
1124                f" getattr(included_template, {name!r}, missing)"
1125            )
1126            self.writeline(f"if {frame.symbols.ref(alias)} is missing:")
1127            self.indent()
1128            message = (
1129                "the template {included_template.__name__!r}"
1130                f" (imported on {self.position(node)})"
1131                f" does not export the requested name {name!r}"
1132            )
1133            self.writeline(
1134                f"{frame.symbols.ref(alias)} = undefined(f{message!r}, name={name!r})"
1135            )
1136            self.outdent()
1137            if frame.toplevel:
1138                var_names.append(alias)
1139                if not alias.startswith("_"):
1140                    discarded_names.append(alias)
1141
1142        if var_names:
1143            if len(var_names) == 1:
1144                name = var_names[0]
1145                self.writeline(f"context.vars[{name!r}] = {frame.symbols.ref(name)}")
1146            else:
1147                names_kv = ", ".join(
1148                    f"{name!r}: {frame.symbols.ref(name)}" for name in var_names
1149                )
1150                self.writeline(f"context.vars.update({{{names_kv}}})")
1151        if discarded_names:
1152            if len(discarded_names) == 1:
1153                self.writeline(f"context.exported_vars.discard({discarded_names[0]!r})")
1154            else:
1155                names_str = ", ".join(map(repr, discarded_names))
1156                self.writeline(
1157                    f"context.exported_vars.difference_update(({names_str}))"
1158                )
1159
1160    def visit_For(self, node: nodes.For, frame: Frame) -> None:
1161        loop_frame = frame.inner()
1162        loop_frame.loop_frame = True
1163        test_frame = frame.inner()
1164        else_frame = frame.inner()
1165
1166        # try to figure out if we have an extended loop.  An extended loop
1167        # is necessary if the loop is in recursive mode if the special loop
1168        # variable is accessed in the body if the body is a scoped block.
1169        extended_loop = (
1170            node.recursive
1171            or "loop"
1172            in find_undeclared(node.iter_child_nodes(only=("body",)), ("loop",))
1173            or any(block.scoped for block in node.find_all(nodes.Block))
1174        )
1175
1176        loop_ref = None
1177        if extended_loop:
1178            loop_ref = loop_frame.symbols.declare_parameter("loop")
1179
1180        loop_frame.symbols.analyze_node(node, for_branch="body")
1181        if node.else_:
1182            else_frame.symbols.analyze_node(node, for_branch="else")
1183
1184        if node.test:
1185            loop_filter_func = self.temporary_identifier()
1186            test_frame.symbols.analyze_node(node, for_branch="test")
1187            self.writeline(f"{self.func(loop_filter_func)}(fiter):", node.test)
1188            self.indent()
1189            self.enter_frame(test_frame)
1190            self.writeline(self.choose_async("async for ", "for "))
1191            self.visit(node.target, loop_frame)
1192            self.write(" in ")
1193            self.write(self.choose_async("auto_aiter(fiter)", "fiter"))
1194            self.write(":")
1195            self.indent()
1196            self.writeline("if ", node.test)
1197            self.visit(node.test, test_frame)
1198            self.write(":")
1199            self.indent()
1200            self.writeline("yield ")
1201            self.visit(node.target, loop_frame)
1202            self.outdent(3)
1203            self.leave_frame(test_frame, with_python_scope=True)
1204
1205        # if we don't have an recursive loop we have to find the shadowed
1206        # variables at that point.  Because loops can be nested but the loop
1207        # variable is a special one we have to enforce aliasing for it.
1208        if node.recursive:
1209            self.writeline(
1210                f"{self.func('loop')}(reciter, loop_render_func, depth=0):", node
1211            )
1212            self.indent()
1213            self.buffer(loop_frame)
1214
1215            # Use the same buffer for the else frame
1216            else_frame.buffer = loop_frame.buffer
1217
1218        # make sure the loop variable is a special one and raise a template
1219        # assertion error if a loop tries to write to loop
1220        if extended_loop:
1221            self.writeline(f"{loop_ref} = missing")
1222
1223        for name in node.find_all(nodes.Name):
1224            if name.ctx == "store" and name.name == "loop":
1225                self.fail(
1226                    "Can't assign to special loop variable in for-loop target",
1227                    name.lineno,
1228                )
1229
1230        if node.else_:
1231            iteration_indicator = self.temporary_identifier()
1232            self.writeline(f"{iteration_indicator} = 1")
1233
1234        self.writeline(self.choose_async("async for ", "for "), node)
1235        self.visit(node.target, loop_frame)
1236        if extended_loop:
1237            self.write(f", {loop_ref} in {self.choose_async('Async')}LoopContext(")
1238        else:
1239            self.write(" in ")
1240
1241        if node.test:
1242            self.write(f"{loop_filter_func}(")
1243        if node.recursive:
1244            self.write("reciter")
1245        else:
1246            if self.environment.is_async and not extended_loop:
1247                self.write("auto_aiter(")
1248            self.visit(node.iter, frame)
1249            if self.environment.is_async and not extended_loop:
1250                self.write(")")
1251        if node.test:
1252            self.write(")")
1253
1254        if node.recursive:
1255            self.write(", undefined, loop_render_func, depth):")
1256        else:
1257            self.write(", undefined):" if extended_loop else ":")
1258
1259        self.indent()
1260        self.enter_frame(loop_frame)
1261
1262        self.writeline("_loop_vars = {}")
1263        self.blockvisit(node.body, loop_frame)
1264        if node.else_:
1265            self.writeline(f"{iteration_indicator} = 0")
1266        self.outdent()
1267        self.leave_frame(
1268            loop_frame, with_python_scope=node.recursive and not node.else_
1269        )
1270
1271        if node.else_:
1272            self.writeline(f"if {iteration_indicator}:")
1273            self.indent()
1274            self.enter_frame(else_frame)
1275            self.blockvisit(node.else_, else_frame)
1276            self.leave_frame(else_frame)
1277            self.outdent()
1278
1279        # if the node was recursive we have to return the buffer contents
1280        # and start the iteration code
1281        if node.recursive:
1282            self.return_buffer_contents(loop_frame)
1283            self.outdent()
1284            self.start_write(frame, node)
1285            self.write(f"{self.choose_async('await ')}loop(")
1286            if self.environment.is_async:
1287                self.write("auto_aiter(")
1288            self.visit(node.iter, frame)
1289            if self.environment.is_async:
1290                self.write(")")
1291            self.write(", loop)")
1292            self.end_write(frame)
1293
1294        # at the end of the iteration, clear any assignments made in the
1295        # loop from the top level
1296        if self._assign_stack:
1297            self._assign_stack[-1].difference_update(loop_frame.symbols.stores)
1298
1299    def visit_If(self, node: nodes.If, frame: Frame) -> None:
1300        if_frame = frame.soft()
1301        self.writeline("if ", node)
1302        self.visit(node.test, if_frame)
1303        self.write(":")
1304        self.indent()
1305        self.blockvisit(node.body, if_frame)
1306        self.outdent()
1307        for elif_ in node.elif_:
1308            self.writeline("elif ", elif_)
1309            self.visit(elif_.test, if_frame)
1310            self.write(":")
1311            self.indent()
1312            self.blockvisit(elif_.body, if_frame)
1313            self.outdent()
1314        if node.else_:
1315            self.writeline("else:")
1316            self.indent()
1317            self.blockvisit(node.else_, if_frame)
1318            self.outdent()
1319
1320    def visit_Macro(self, node: nodes.Macro, frame: Frame) -> None:
1321        macro_frame, macro_ref = self.macro_body(node, frame)
1322        self.newline()
1323        if frame.toplevel:
1324            if not node.name.startswith("_"):
1325                self.write(f"context.exported_vars.add({node.name!r})")
1326            self.writeline(f"context.vars[{node.name!r}] = ")
1327        self.write(f"{frame.symbols.ref(node.name)} = ")
1328        self.macro_def(macro_ref, macro_frame)
1329
1330    def visit_CallBlock(self, node: nodes.CallBlock, frame: Frame) -> None:
1331        call_frame, macro_ref = self.macro_body(node, frame)
1332        self.writeline("caller = ")
1333        self.macro_def(macro_ref, call_frame)
1334        self.start_write(frame, node)
1335        self.visit_Call(node.call, frame, forward_caller=True)
1336        self.end_write(frame)
1337
1338    def visit_FilterBlock(self, node: nodes.FilterBlock, frame: Frame) -> None:
1339        filter_frame = frame.inner()
1340        filter_frame.symbols.analyze_node(node)
1341        self.enter_frame(filter_frame)
1342        self.buffer(filter_frame)
1343        self.blockvisit(node.body, filter_frame)
1344        self.start_write(frame, node)
1345        self.visit_Filter(node.filter, filter_frame)
1346        self.end_write(frame)
1347        self.leave_frame(filter_frame)
1348
1349    def visit_With(self, node: nodes.With, frame: Frame) -> None:
1350        with_frame = frame.inner()
1351        with_frame.symbols.analyze_node(node)
1352        self.enter_frame(with_frame)
1353        for target, expr in zip(node.targets, node.values):
1354            self.newline()
1355            self.visit(target, with_frame)
1356            self.write(" = ")
1357            self.visit(expr, frame)
1358        self.blockvisit(node.body, with_frame)
1359        self.leave_frame(with_frame)
1360
1361    def visit_ExprStmt(self, node: nodes.ExprStmt, frame: Frame) -> None:
1362        self.newline(node)
1363        self.visit(node.node, frame)
1364
1365    class _FinalizeInfo(t.NamedTuple):
1366        const: t.Optional[t.Callable[..., str]]
1367        src: t.Optional[str]
1368
1369    @staticmethod
1370    def _default_finalize(value: t.Any) -> t.Any:
1371        """The default finalize function if the environment isn't
1372        configured with one. Or, if the environment has one, this is
1373        called on that function's output for constants.
1374        """
1375        return str(value)
1376
1377    _finalize: t.Optional[_FinalizeInfo] = None
1378
1379    def _make_finalize(self) -> _FinalizeInfo:
1380        """Build the finalize function to be used on constants and at
1381        runtime. Cached so it's only created once for all output nodes.
1382
1383        Returns a ``namedtuple`` with the following attributes:
1384
1385        ``const``
1386            A function to finalize constant data at compile time.
1387
1388        ``src``
1389            Source code to output around nodes to be evaluated at
1390            runtime.
1391        """
1392        if self._finalize is not None:
1393            return self._finalize
1394
1395        finalize: t.Optional[t.Callable[..., t.Any]]
1396        finalize = default = self._default_finalize
1397        src = None
1398
1399        if self.environment.finalize:
1400            src = "environment.finalize("
1401            env_finalize = self.environment.finalize
1402            pass_arg = {
1403                _PassArg.context: "context",
1404                _PassArg.eval_context: "context.eval_ctx",
1405                _PassArg.environment: "environment",
1406            }.get(
1407                _PassArg.from_obj(env_finalize)  # type: ignore
1408            )
1409            finalize = None
1410
1411            if pass_arg is None:
1412
1413                def finalize(value: t.Any) -> t.Any:  # noqa: F811
1414                    return default(env_finalize(value))
1415
1416            else:
1417                src = f"{src}{pass_arg}, "
1418
1419                if pass_arg == "environment":
1420
1421                    def finalize(value: t.Any) -> t.Any:  # noqa: F811
1422                        return default(env_finalize(self.environment, value))
1423
1424        self._finalize = self._FinalizeInfo(finalize, src)
1425        return self._finalize
1426
1427    def _output_const_repr(self, group: t.Iterable[t.Any]) -> str:
1428        """Given a group of constant values converted from ``Output``
1429        child nodes, produce a string to write to the template module
1430        source.
1431        """
1432        return repr(concat(group))
1433
1434    def _output_child_to_const(
1435        self, node: nodes.Expr, frame: Frame, finalize: _FinalizeInfo
1436    ) -> str:
1437        """Try to optimize a child of an ``Output`` node by trying to
1438        convert it to constant, finalized data at compile time.
1439
1440        If :exc:`Impossible` is raised, the node is not constant and
1441        will be evaluated at runtime. Any other exception will also be
1442        evaluated at runtime for easier debugging.
1443        """
1444        const = node.as_const(frame.eval_ctx)
1445
1446        if frame.eval_ctx.autoescape:
1447            const = escape(const)
1448
1449        # Template data doesn't go through finalize.
1450        if isinstance(node, nodes.TemplateData):
1451            return str(const)
1452
1453        return finalize.const(const)  # type: ignore
1454
1455    def _output_child_pre(
1456        self, node: nodes.Expr, frame: Frame, finalize: _FinalizeInfo
1457    ) -> None:
1458        """Output extra source code before visiting a child of an
1459        ``Output`` node.
1460        """
1461        if frame.eval_ctx.volatile:
1462            self.write("(escape if context.eval_ctx.autoescape else str)(")
1463        elif frame.eval_ctx.autoescape:
1464            self.write("escape(")
1465        else:
1466            self.write("str(")
1467
1468        if finalize.src is not None:
1469            self.write(finalize.src)
1470
1471    def _output_child_post(
1472        self, node: nodes.Expr, frame: Frame, finalize: _FinalizeInfo
1473    ) -> None:
1474        """Output extra source code after visiting a child of an
1475        ``Output`` node.
1476        """
1477        self.write(")")
1478
1479        if finalize.src is not None:
1480            self.write(")")
1481
1482    def visit_Output(self, node: nodes.Output, frame: Frame) -> None:
1483        # If an extends is active, don't render outside a block.
1484        if frame.require_output_check:
1485            # A top-level extends is known to exist at compile time.
1486            if self.has_known_extends:
1487                return
1488
1489            self.writeline("if parent_template is None:")
1490            self.indent()
1491
1492        finalize = self._make_finalize()
1493        body: t.List[t.Union[t.List[t.Any], nodes.Expr]] = []
1494
1495        # Evaluate constants at compile time if possible. Each item in
1496        # body will be either a list of static data or a node to be
1497        # evaluated at runtime.
1498        for child in node.nodes:
1499            try:
1500                if not (
1501                    # If the finalize function requires runtime context,
1502                    # constants can't be evaluated at compile time.
1503                    finalize.const
1504                    # Unless it's basic template data that won't be
1505                    # finalized anyway.
1506                    or isinstance(child, nodes.TemplateData)
1507                ):
1508                    raise nodes.Impossible()
1509
1510                const = self._output_child_to_const(child, frame, finalize)
1511            except (nodes.Impossible, Exception):
1512                # The node was not constant and needs to be evaluated at
1513                # runtime. Or another error was raised, which is easier
1514                # to debug at runtime.
1515                body.append(child)
1516                continue
1517
1518            if body and isinstance(body[-1], list):
1519                body[-1].append(const)
1520            else:
1521                body.append([const])
1522
1523        if frame.buffer is not None:
1524            if len(body) == 1:
1525                self.writeline(f"{frame.buffer}.append(")
1526            else:
1527                self.writeline(f"{frame.buffer}.extend((")
1528
1529            self.indent()
1530
1531        for item in body:
1532            if isinstance(item, list):
1533                # A group of constant data to join and output.
1534                val = self._output_const_repr(item)
1535
1536                if frame.buffer is None:
1537                    self.writeline("yield " + val)
1538                else:
1539                    self.writeline(val + ",")
1540            else:
1541                if frame.buffer is None:
1542                    self.writeline("yield ", item)
1543                else:
1544                    self.newline(item)
1545
1546                # A node to be evaluated at runtime.
1547                self._output_child_pre(item, frame, finalize)
1548                self.visit(item, frame)
1549                self._output_child_post(item, frame, finalize)
1550
1551                if frame.buffer is not None:
1552                    self.write(",")
1553
1554        if frame.buffer is not None:
1555            self.outdent()
1556            self.writeline(")" if len(body) == 1 else "))")
1557
1558        if frame.require_output_check:
1559            self.outdent()
1560
1561    def visit_Assign(self, node: nodes.Assign, frame: Frame) -> None:
1562        self.push_assign_tracking()
1563        self.newline(node)
1564        self.visit(node.target, frame)
1565        self.write(" = ")
1566        self.visit(node.node, frame)
1567        self.pop_assign_tracking(frame)
1568
1569    def visit_AssignBlock(self, node: nodes.AssignBlock, frame: Frame) -> None:
1570        self.push_assign_tracking()
1571        block_frame = frame.inner()
1572        # This is a special case.  Since a set block always captures we
1573        # will disable output checks.  This way one can use set blocks
1574        # toplevel even in extended templates.
1575        block_frame.require_output_check = False
1576        block_frame.symbols.analyze_node(node)
1577        self.enter_frame(block_frame)
1578        self.buffer(block_frame)
1579        self.blockvisit(node.body, block_frame)
1580        self.newline(node)
1581        self.visit(node.target, frame)
1582        self.write(" = (Markup if context.eval_ctx.autoescape else identity)(")
1583        if node.filter is not None:
1584            self.visit_Filter(node.filter, block_frame)
1585        else:
1586            self.write(f"concat({block_frame.buffer})")
1587        self.write(")")
1588        self.pop_assign_tracking(frame)
1589        self.leave_frame(block_frame)
1590
1591    # -- Expression Visitors
1592
1593    def visit_Name(self, node: nodes.Name, frame: Frame) -> None:
1594        if node.ctx == "store" and (
1595            frame.toplevel or frame.loop_frame or frame.block_frame
1596        ):
1597            if self._assign_stack:
1598                self._assign_stack[-1].add(node.name)
1599        ref = frame.symbols.ref(node.name)
1600
1601        # If we are looking up a variable we might have to deal with the
1602        # case where it's undefined.  We can skip that case if the load
1603        # instruction indicates a parameter which are always defined.
1604        if node.ctx == "load":
1605            load = frame.symbols.find_load(ref)
1606            if not (
1607                load is not None
1608                and load[0] == VAR_LOAD_PARAMETER
1609                and not self.parameter_is_undeclared(ref)
1610            ):
1611                self.write(
1612                    f"(undefined(name={node.name!r}) if {ref} is missing else {ref})"
1613                )
1614                return
1615
1616        self.write(ref)
1617
1618    def visit_NSRef(self, node: nodes.NSRef, frame: Frame) -> None:
1619        # NSRefs can only be used to store values; since they use the normal
1620        # `foo.bar` notation they will be parsed as a normal attribute access
1621        # when used anywhere but in a `set` context
1622        ref = frame.symbols.ref(node.name)
1623        self.writeline(f"if not isinstance({ref}, Namespace):")
1624        self.indent()
1625        self.writeline(
1626            "raise TemplateRuntimeError"
1627            '("cannot assign attribute on non-namespace object")'
1628        )
1629        self.outdent()
1630        self.writeline(f"{ref}[{node.attr!r}]")
1631
1632    def visit_Const(self, node: nodes.Const, frame: Frame) -> None:
1633        val = node.as_const(frame.eval_ctx)
1634        if isinstance(val, float):
1635            self.write(str(val))
1636        else:
1637            self.write(repr(val))
1638
1639    def visit_TemplateData(self, node: nodes.TemplateData, frame: Frame) -> None:
1640        try:
1641            self.write(repr(node.as_const(frame.eval_ctx)))
1642        except nodes.Impossible:
1643            self.write(
1644                f"(Markup if context.eval_ctx.autoescape else identity)({node.data!r})"
1645            )
1646
1647    def visit_Tuple(self, node: nodes.Tuple, frame: Frame) -> None:
1648        self.write("(")
1649        idx = -1
1650        for idx, item in enumerate(node.items):
1651            if idx:
1652                self.write(", ")
1653            self.visit(item, frame)
1654        self.write(",)" if idx == 0 else ")")
1655
1656    def visit_List(self, node: nodes.List, frame: Frame) -> None:
1657        self.write("[")
1658        for idx, item in enumerate(node.items):
1659            if idx:
1660                self.write(", ")
1661            self.visit(item, frame)
1662        self.write("]")
1663
1664    def visit_Dict(self, node: nodes.Dict, frame: Frame) -> None:
1665        self.write("{")
1666        for idx, item in enumerate(node.items):
1667            if idx:
1668                self.write(", ")
1669            self.visit(item.key, frame)
1670            self.write(": ")
1671            self.visit(item.value, frame)
1672        self.write("}")
1673
1674    visit_Add = _make_binop("+")
1675    visit_Sub = _make_binop("-")
1676    visit_Mul = _make_binop("*")
1677    visit_Div = _make_binop("/")
1678    visit_FloorDiv = _make_binop("//")
1679    visit_Pow = _make_binop("**")
1680    visit_Mod = _make_binop("%")
1681    visit_And = _make_binop("and")
1682    visit_Or = _make_binop("or")
1683    visit_Pos = _make_unop("+")
1684    visit_Neg = _make_unop("-")
1685    visit_Not = _make_unop("not ")
1686
1687    @optimizeconst
1688    def visit_Concat(self, node: nodes.Concat, frame: Frame) -> None:
1689        if frame.eval_ctx.volatile:
1690            func_name = "(markup_join if context.eval_ctx.volatile else str_join)"
1691        elif frame.eval_ctx.autoescape:
1692            func_name = "markup_join"
1693        else:
1694            func_name = "str_join"
1695        self.write(f"{func_name}((")
1696        for arg in node.nodes:
1697            self.visit(arg, frame)
1698            self.write(", ")
1699        self.write("))")
1700
1701    @optimizeconst
1702    def visit_Compare(self, node: nodes.Compare, frame: Frame) -> None:
1703        self.write("(")
1704        self.visit(node.expr, frame)
1705        for op in node.ops:
1706            self.visit(op, frame)
1707        self.write(")")
1708
1709    def visit_Operand(self, node: nodes.Operand, frame: Frame) -> None:
1710        self.write(f" {operators[node.op]} ")
1711        self.visit(node.expr, frame)
1712
1713    @optimizeconst
1714    def visit_Getattr(self, node: nodes.Getattr, frame: Frame) -> None:
1715        if self.environment.is_async:
1716            self.write("(await auto_await(")
1717
1718        self.write("environment.getattr(")
1719        self.visit(node.node, frame)
1720        self.write(f", {node.attr!r})")
1721
1722        if self.environment.is_async:
1723            self.write("))")
1724
1725    @optimizeconst
1726    def visit_Getitem(self, node: nodes.Getitem, frame: Frame) -> None:
1727        # slices bypass the environment getitem method.
1728        if isinstance(node.arg, nodes.Slice):
1729            self.visit(node.node, frame)
1730            self.write("[")
1731            self.visit(node.arg, frame)
1732            self.write("]")
1733        else:
1734            if self.environment.is_async:
1735                self.write("(await auto_await(")
1736
1737            self.write("environment.getitem(")
1738            self.visit(node.node, frame)
1739            self.write(", ")
1740            self.visit(node.arg, frame)
1741            self.write(")")
1742
1743            if self.environment.is_async:
1744                self.write("))")
1745
1746    def visit_Slice(self, node: nodes.Slice, frame: Frame) -> None:
1747        if node.start is not None:
1748            self.visit(node.start, frame)
1749        self.write(":")
1750        if node.stop is not None:
1751            self.visit(node.stop, frame)
1752        if node.step is not None:
1753            self.write(":")
1754            self.visit(node.step, frame)
1755
1756    @contextmanager
1757    def _filter_test_common(
1758        self, node: t.Union[nodes.Filter, nodes.Test], frame: Frame, is_filter: bool
1759    ) -> t.Iterator[None]:
1760        if self.environment.is_async:
1761            self.write("(await auto_await(")
1762
1763        if is_filter:
1764            self.write(f"{self.filters[node.name]}(")
1765            func = self.environment.filters.get(node.name)
1766        else:
1767            self.write(f"{self.tests[node.name]}(")
1768            func = self.environment.tests.get(node.name)
1769
1770        # When inside an If or CondExpr frame, allow the filter to be
1771        # undefined at compile time and only raise an error if it's
1772        # actually called at runtime. See pull_dependencies.
1773        if func is None and not frame.soft_frame:
1774            type_name = "filter" if is_filter else "test"
1775            self.fail(f"No {type_name} named {node.name!r}.", node.lineno)
1776
1777        pass_arg = {
1778            _PassArg.context: "context",
1779            _PassArg.eval_context: "context.eval_ctx",
1780            _PassArg.environment: "environment",
1781        }.get(
1782            _PassArg.from_obj(func)  # type: ignore
1783        )
1784
1785        if pass_arg is not None:
1786            self.write(f"{pass_arg}, ")
1787
1788        # Back to the visitor function to handle visiting the target of
1789        # the filter or test.
1790        yield
1791
1792        self.signature(node, frame)
1793        self.write(")")
1794
1795        if self.environment.is_async:
1796            self.write("))")
1797
1798    @optimizeconst
1799    def visit_Filter(self, node: nodes.Filter, frame: Frame) -> None:
1800        with self._filter_test_common(node, frame, True):
1801            # if the filter node is None we are inside a filter block
1802            # and want to write to the current buffer
1803            if node.node is not None:
1804                self.visit(node.node, frame)
1805            elif frame.eval_ctx.volatile:
1806                self.write(
1807                    f"(Markup(concat({frame.buffer}))"
1808                    f" if context.eval_ctx.autoescape else concat({frame.buffer}))"
1809                )
1810            elif frame.eval_ctx.autoescape:
1811                self.write(f"Markup(concat({frame.buffer}))")
1812            else:
1813                self.write(f"concat({frame.buffer})")
1814
1815    @optimizeconst
1816    def visit_Test(self, node: nodes.Test, frame: Frame) -> None:
1817        with self._filter_test_common(node, frame, False):
1818            self.visit(node.node, frame)
1819
1820    @optimizeconst
1821    def visit_CondExpr(self, node: nodes.CondExpr, frame: Frame) -> None:
1822        frame = frame.soft()
1823
1824        def write_expr2() -> None:
1825            if node.expr2 is not None:
1826                self.visit(node.expr2, frame)
1827                return
1828
1829            self.write(
1830                f'cond_expr_undefined("the inline if-expression on'
1831                f" {self.position(node)} evaluated to false and no else"
1832                f' section was defined.")'
1833            )
1834
1835        self.write("(")
1836        self.visit(node.expr1, frame)
1837        self.write(" if ")
1838        self.visit(node.test, frame)
1839        self.write(" else ")
1840        write_expr2()
1841        self.write(")")
1842
1843    @optimizeconst
1844    def visit_Call(
1845        self, node: nodes.Call, frame: Frame, forward_caller: bool = False
1846    ) -> None:
1847        if self.environment.is_async:
1848            self.write("(await auto_await(")
1849        if self.environment.sandboxed:
1850            self.write("environment.call(context, ")
1851        else:
1852            self.write("context.call(")
1853        self.visit(node.node, frame)
1854        extra_kwargs = {"caller": "caller"} if forward_caller else None
1855        loop_kwargs = {"_loop_vars": "_loop_vars"} if frame.loop_frame else {}
1856        block_kwargs = {"_block_vars": "_block_vars"} if frame.block_frame else {}
1857        if extra_kwargs:
1858            extra_kwargs.update(loop_kwargs, **block_kwargs)
1859        elif loop_kwargs or block_kwargs:
1860            extra_kwargs = dict(loop_kwargs, **block_kwargs)
1861        self.signature(node, frame, extra_kwargs)
1862        self.write(")")
1863        if self.environment.is_async:
1864            self.write("))")
1865
1866    def visit_Keyword(self, node: nodes.Keyword, frame: Frame) -> None:
1867        self.write(node.key + "=")
1868        self.visit(node.value, frame)
1869
1870    # -- Unused nodes for extensions
1871
1872    def visit_MarkSafe(self, node: nodes.MarkSafe, frame: Frame) -> None:
1873        self.write("Markup(")
1874        self.visit(node.expr, frame)
1875        self.write(")")
1876
1877    def visit_MarkSafeIfAutoescape(
1878        self, node: nodes.MarkSafeIfAutoescape, frame: Frame
1879    ) -> None:
1880        self.write("(Markup if context.eval_ctx.autoescape else identity)(")
1881        self.visit(node.expr, frame)
1882        self.write(")")
1883
1884    def visit_EnvironmentAttribute(
1885        self, node: nodes.EnvironmentAttribute, frame: Frame
1886    ) -> None:
1887        self.write("environment." + node.name)
1888
1889    def visit_ExtensionAttribute(
1890        self, node: nodes.ExtensionAttribute, frame: Frame
1891    ) -> None:
1892        self.write(f"environment.extensions[{node.identifier!r}].{node.name}")
1893
1894    def visit_ImportedName(self, node: nodes.ImportedName, frame: Frame) -> None:
1895        self.write(self.import_aliases[node.importname])
1896
1897    def visit_InternalName(self, node: nodes.InternalName, frame: Frame) -> None:
1898        self.write(node.name)
1899
1900    def visit_ContextReference(
1901        self, node: nodes.ContextReference, frame: Frame
1902    ) -> None:
1903        self.write("context")
1904
1905    def visit_DerivedContextReference(
1906        self, node: nodes.DerivedContextReference, frame: Frame
1907    ) -> None:
1908        self.write(self.derive_context(frame))
1909
1910    def visit_Continue(self, node: nodes.Continue, frame: Frame) -> None:
1911        self.writeline("continue", node)
1912
1913    def visit_Break(self, node: nodes.Break, frame: Frame) -> None:
1914        self.writeline("break", node)
1915
1916    def visit_Scope(self, node: nodes.Scope, frame: Frame) -> None:
1917        scope_frame = frame.inner()
1918        scope_frame.symbols.analyze_node(node)
1919        self.enter_frame(scope_frame)
1920        self.blockvisit(node.body, scope_frame)
1921        self.leave_frame(scope_frame)
1922
1923    def visit_OverlayScope(self, node: nodes.OverlayScope, frame: Frame) -> None:
1924        ctx = self.temporary_identifier()
1925        self.writeline(f"{ctx} = {self.derive_context(frame)}")
1926        self.writeline(f"{ctx}.vars = ")
1927        self.visit(node.context, frame)
1928        self.push_context_reference(ctx)
1929
1930        scope_frame = frame.inner(isolated=True)
1931        scope_frame.symbols.analyze_node(node)
1932        self.enter_frame(scope_frame)
1933        self.blockvisit(node.body, scope_frame)
1934        self.leave_frame(scope_frame)
1935        self.pop_context_reference()
1936
1937    def visit_EvalContextModifier(
1938        self, node: nodes.EvalContextModifier, frame: Frame
1939    ) -> None:
1940        for keyword in node.options:
1941            self.writeline(f"context.eval_ctx.{keyword.key} = ")
1942            self.visit(keyword.value, frame)
1943            try:
1944                val = keyword.value.as_const(frame.eval_ctx)
1945            except nodes.Impossible:
1946                frame.eval_ctx.volatile = True
1947            else:
1948                setattr(frame.eval_ctx, keyword.key, val)
1949
1950    def visit_ScopedEvalContextModifier(
1951        self, node: nodes.ScopedEvalContextModifier, frame: Frame
1952    ) -> None:
1953        old_ctx_name = self.temporary_identifier()
1954        saved_ctx = frame.eval_ctx.save()
1955        self.writeline(f"{old_ctx_name} = context.eval_ctx.save()")
1956        self.visit_EvalContextModifier(node, frame)
1957        for child in node.body:
1958            self.visit(child, frame)
1959        frame.eval_ctx.revert(saved_ctx)
1960        self.writeline(f"context.eval_ctx.revert({old_ctx_name})")