jinja2.ext

  1"""Extension API for adding custom tags and behavior."""
  2import pprint
  3import re
  4import typing as t
  5
  6from markupsafe import Markup
  7
  8from . import defaults
  9from . import nodes
 10from .environment import Environment
 11from .exceptions import TemplateAssertionError
 12from .exceptions import TemplateSyntaxError
 13from .runtime import concat  # type: ignore
 14from .runtime import Context
 15from .runtime import Undefined
 16from .utils import import_string
 17from .utils import pass_context
 18
 19if t.TYPE_CHECKING:
 20    import typing_extensions as te
 21    from .lexer import Token
 22    from .lexer import TokenStream
 23    from .parser import Parser
 24
 25    class _TranslationsBasic(te.Protocol):
 26        def gettext(self, message: str) -> str:
 27            ...
 28
 29        def ngettext(self, singular: str, plural: str, n: int) -> str:
 30            pass
 31
 32    class _TranslationsContext(_TranslationsBasic):
 33        def pgettext(self, context: str, message: str) -> str:
 34            ...
 35
 36        def npgettext(self, context: str, singular: str, plural: str, n: int) -> str:
 37            ...
 38
 39    _SupportedTranslations = t.Union[_TranslationsBasic, _TranslationsContext]
 40
 41
 42# I18N functions available in Jinja templates. If the I18N library
 43# provides ugettext, it will be assigned to gettext.
 44GETTEXT_FUNCTIONS: t.Tuple[str, ...] = (
 45    "_",
 46    "gettext",
 47    "ngettext",
 48    "pgettext",
 49    "npgettext",
 50)
 51_ws_re = re.compile(r"\s*\n\s*")
 52
 53
 54class Extension:
 55    """Extensions can be used to add extra functionality to the Jinja template
 56    system at the parser level.  Custom extensions are bound to an environment
 57    but may not store environment specific data on `self`.  The reason for
 58    this is that an extension can be bound to another environment (for
 59    overlays) by creating a copy and reassigning the `environment` attribute.
 60
 61    As extensions are created by the environment they cannot accept any
 62    arguments for configuration.  One may want to work around that by using
 63    a factory function, but that is not possible as extensions are identified
 64    by their import name.  The correct way to configure the extension is
 65    storing the configuration values on the environment.  Because this way the
 66    environment ends up acting as central configuration storage the
 67    attributes may clash which is why extensions have to ensure that the names
 68    they choose for configuration are not too generic.  ``prefix`` for example
 69    is a terrible name, ``fragment_cache_prefix`` on the other hand is a good
 70    name as includes the name of the extension (fragment cache).
 71    """
 72
 73    identifier: t.ClassVar[str]
 74
 75    def __init_subclass__(cls) -> None:
 76        cls.identifier = f"{cls.__module__}.{cls.__name__}"
 77
 78    #: if this extension parses this is the list of tags it's listening to.
 79    tags: t.Set[str] = set()
 80
 81    #: the priority of that extension.  This is especially useful for
 82    #: extensions that preprocess values.  A lower value means higher
 83    #: priority.
 84    #:
 85    #: .. versionadded:: 2.4
 86    priority = 100
 87
 88    def __init__(self, environment: Environment) -> None:
 89        self.environment = environment
 90
 91    def bind(self, environment: Environment) -> "Extension":
 92        """Create a copy of this extension bound to another environment."""
 93        rv = object.__new__(self.__class__)
 94        rv.__dict__.update(self.__dict__)
 95        rv.environment = environment
 96        return rv
 97
 98    def preprocess(
 99        self, source: str, name: t.Optional[str], filename: t.Optional[str] = None
100    ) -> str:
101        """This method is called before the actual lexing and can be used to
102        preprocess the source.  The `filename` is optional.  The return value
103        must be the preprocessed source.
104        """
105        return source
106
107    def filter_stream(
108        self, stream: "TokenStream"
109    ) -> t.Union["TokenStream", t.Iterable["Token"]]:
110        """It's passed a :class:`~jinja2.lexer.TokenStream` that can be used
111        to filter tokens returned.  This method has to return an iterable of
112        :class:`~jinja2.lexer.Token`\\s, but it doesn't have to return a
113        :class:`~jinja2.lexer.TokenStream`.
114        """
115        return stream
116
117    def parse(self, parser: "Parser") -> t.Union[nodes.Node, t.List[nodes.Node]]:
118        """If any of the :attr:`tags` matched this method is called with the
119        parser as first argument.  The token the parser stream is pointing at
120        is the name token that matched.  This method has to return one or a
121        list of multiple nodes.
122        """
123        raise NotImplementedError()
124
125    def attr(
126        self, name: str, lineno: t.Optional[int] = None
127    ) -> nodes.ExtensionAttribute:
128        """Return an attribute node for the current extension.  This is useful
129        to pass constants on extensions to generated template code.
130
131        ::
132
133            self.attr('_my_attribute', lineno=lineno)
134        """
135        return nodes.ExtensionAttribute(self.identifier, name, lineno=lineno)
136
137    def call_method(
138        self,
139        name: str,
140        args: t.Optional[t.List[nodes.Expr]] = None,
141        kwargs: t.Optional[t.List[nodes.Keyword]] = None,
142        dyn_args: t.Optional[nodes.Expr] = None,
143        dyn_kwargs: t.Optional[nodes.Expr] = None,
144        lineno: t.Optional[int] = None,
145    ) -> nodes.Call:
146        """Call a method of the extension.  This is a shortcut for
147        :meth:`attr` + :class:`jinja2.nodes.Call`.
148        """
149        if args is None:
150            args = []
151        if kwargs is None:
152            kwargs = []
153        return nodes.Call(
154            self.attr(name, lineno=lineno),
155            args,
156            kwargs,
157            dyn_args,
158            dyn_kwargs,
159            lineno=lineno,
160        )
161
162
163@pass_context
164def _gettext_alias(
165    __context: Context, *args: t.Any, **kwargs: t.Any
166) -> t.Union[t.Any, Undefined]:
167    return __context.call(__context.resolve("gettext"), *args, **kwargs)
168
169
170def _make_new_gettext(func: t.Callable[[str], str]) -> t.Callable[..., str]:
171    @pass_context
172    def gettext(__context: Context, __string: str, **variables: t.Any) -> str:
173        rv = __context.call(func, __string)
174        if __context.eval_ctx.autoescape:
175            rv = Markup(rv)
176        # Always treat as a format string, even if there are no
177        # variables. This makes translation strings more consistent
178        # and predictable. This requires escaping
179        return rv % variables  # type: ignore
180
181    return gettext
182
183
184def _make_new_ngettext(func: t.Callable[[str, str, int], str]) -> t.Callable[..., str]:
185    @pass_context
186    def ngettext(
187        __context: Context,
188        __singular: str,
189        __plural: str,
190        __num: int,
191        **variables: t.Any,
192    ) -> str:
193        variables.setdefault("num", __num)
194        rv = __context.call(func, __singular, __plural, __num)
195        if __context.eval_ctx.autoescape:
196            rv = Markup(rv)
197        # Always treat as a format string, see gettext comment above.
198        return rv % variables  # type: ignore
199
200    return ngettext
201
202
203def _make_new_pgettext(func: t.Callable[[str, str], str]) -> t.Callable[..., str]:
204    @pass_context
205    def pgettext(
206        __context: Context, __string_ctx: str, __string: str, **variables: t.Any
207    ) -> str:
208        variables.setdefault("context", __string_ctx)
209        rv = __context.call(func, __string_ctx, __string)
210
211        if __context.eval_ctx.autoescape:
212            rv = Markup(rv)
213
214        # Always treat as a format string, see gettext comment above.
215        return rv % variables  # type: ignore
216
217    return pgettext
218
219
220def _make_new_npgettext(
221    func: t.Callable[[str, str, str, int], str]
222) -> t.Callable[..., str]:
223    @pass_context
224    def npgettext(
225        __context: Context,
226        __string_ctx: str,
227        __singular: str,
228        __plural: str,
229        __num: int,
230        **variables: t.Any,
231    ) -> str:
232        variables.setdefault("context", __string_ctx)
233        variables.setdefault("num", __num)
234        rv = __context.call(func, __string_ctx, __singular, __plural, __num)
235
236        if __context.eval_ctx.autoescape:
237            rv = Markup(rv)
238
239        # Always treat as a format string, see gettext comment above.
240        return rv % variables  # type: ignore
241
242    return npgettext
243
244
245class InternationalizationExtension(Extension):
246    """This extension adds gettext support to Jinja."""
247
248    tags = {"trans"}
249
250    # TODO: the i18n extension is currently reevaluating values in a few
251    # situations.  Take this example:
252    #   {% trans count=something() %}{{ count }} foo{% pluralize
253    #     %}{{ count }} fooss{% endtrans %}
254    # something is called twice here.  One time for the gettext value and
255    # the other time for the n-parameter of the ngettext function.
256
257    def __init__(self, environment: Environment) -> None:
258        super().__init__(environment)
259        environment.globals["_"] = _gettext_alias
260        environment.extend(
261            install_gettext_translations=self._install,
262            install_null_translations=self._install_null,
263            install_gettext_callables=self._install_callables,
264            uninstall_gettext_translations=self._uninstall,
265            extract_translations=self._extract,
266            newstyle_gettext=False,
267        )
268
269    def _install(
270        self, translations: "_SupportedTranslations", newstyle: t.Optional[bool] = None
271    ) -> None:
272        # ugettext and ungettext are preferred in case the I18N library
273        # is providing compatibility with older Python versions.
274        gettext = getattr(translations, "ugettext", None)
275        if gettext is None:
276            gettext = translations.gettext
277        ngettext = getattr(translations, "ungettext", None)
278        if ngettext is None:
279            ngettext = translations.ngettext
280
281        pgettext = getattr(translations, "pgettext", None)
282        npgettext = getattr(translations, "npgettext", None)
283        self._install_callables(
284            gettext, ngettext, newstyle=newstyle, pgettext=pgettext, npgettext=npgettext
285        )
286
287    def _install_null(self, newstyle: t.Optional[bool] = None) -> None:
288        import gettext
289
290        translations = gettext.NullTranslations()
291
292        if hasattr(translations, "pgettext"):
293            # Python < 3.8
294            pgettext = translations.pgettext
295        else:
296
297            def pgettext(c: str, s: str) -> str:
298                return s
299
300        if hasattr(translations, "npgettext"):
301            npgettext = translations.npgettext
302        else:
303
304            def npgettext(c: str, s: str, p: str, n: int) -> str:
305                return s if n == 1 else p
306
307        self._install_callables(
308            gettext=translations.gettext,
309            ngettext=translations.ngettext,
310            newstyle=newstyle,
311            pgettext=pgettext,
312            npgettext=npgettext,
313        )
314
315    def _install_callables(
316        self,
317        gettext: t.Callable[[str], str],
318        ngettext: t.Callable[[str, str, int], str],
319        newstyle: t.Optional[bool] = None,
320        pgettext: t.Optional[t.Callable[[str, str], str]] = None,
321        npgettext: t.Optional[t.Callable[[str, str, str, int], str]] = None,
322    ) -> None:
323        if newstyle is not None:
324            self.environment.newstyle_gettext = newstyle  # type: ignore
325        if self.environment.newstyle_gettext:  # type: ignore
326            gettext = _make_new_gettext(gettext)
327            ngettext = _make_new_ngettext(ngettext)
328
329            if pgettext is not None:
330                pgettext = _make_new_pgettext(pgettext)
331
332            if npgettext is not None:
333                npgettext = _make_new_npgettext(npgettext)
334
335        self.environment.globals.update(
336            gettext=gettext, ngettext=ngettext, pgettext=pgettext, npgettext=npgettext
337        )
338
339    def _uninstall(self, translations: "_SupportedTranslations") -> None:
340        for key in ("gettext", "ngettext", "pgettext", "npgettext"):
341            self.environment.globals.pop(key, None)
342
343    def _extract(
344        self,
345        source: t.Union[str, nodes.Template],
346        gettext_functions: t.Sequence[str] = GETTEXT_FUNCTIONS,
347    ) -> t.Iterator[
348        t.Tuple[int, str, t.Union[t.Optional[str], t.Tuple[t.Optional[str], ...]]]
349    ]:
350        if isinstance(source, str):
351            source = self.environment.parse(source)
352        return extract_from_ast(source, gettext_functions)
353
354    def parse(self, parser: "Parser") -> t.Union[nodes.Node, t.List[nodes.Node]]:
355        """Parse a translatable tag."""
356        lineno = next(parser.stream).lineno
357
358        context = None
359        context_token = parser.stream.next_if("string")
360
361        if context_token is not None:
362            context = context_token.value
363
364        # find all the variables referenced.  Additionally a variable can be
365        # defined in the body of the trans block too, but this is checked at
366        # a later state.
367        plural_expr: t.Optional[nodes.Expr] = None
368        plural_expr_assignment: t.Optional[nodes.Assign] = None
369        num_called_num = False
370        variables: t.Dict[str, nodes.Expr] = {}
371        trimmed = None
372        while parser.stream.current.type != "block_end":
373            if variables:
374                parser.stream.expect("comma")
375
376            # skip colon for python compatibility
377            if parser.stream.skip_if("colon"):
378                break
379
380            token = parser.stream.expect("name")
381            if token.value in variables:
382                parser.fail(
383                    f"translatable variable {token.value!r} defined twice.",
384                    token.lineno,
385                    exc=TemplateAssertionError,
386                )
387
388            # expressions
389            if parser.stream.current.type == "assign":
390                next(parser.stream)
391                variables[token.value] = var = parser.parse_expression()
392            elif trimmed is None and token.value in ("trimmed", "notrimmed"):
393                trimmed = token.value == "trimmed"
394                continue
395            else:
396                variables[token.value] = var = nodes.Name(token.value, "load")
397
398            if plural_expr is None:
399                if isinstance(var, nodes.Call):
400                    plural_expr = nodes.Name("_trans", "load")
401                    variables[token.value] = plural_expr
402                    plural_expr_assignment = nodes.Assign(
403                        nodes.Name("_trans", "store"), var
404                    )
405                else:
406                    plural_expr = var
407                num_called_num = token.value == "num"
408
409        parser.stream.expect("block_end")
410
411        plural = None
412        have_plural = False
413        referenced = set()
414
415        # now parse until endtrans or pluralize
416        singular_names, singular = self._parse_block(parser, True)
417        if singular_names:
418            referenced.update(singular_names)
419            if plural_expr is None:
420                plural_expr = nodes.Name(singular_names[0], "load")
421                num_called_num = singular_names[0] == "num"
422
423        # if we have a pluralize block, we parse that too
424        if parser.stream.current.test("name:pluralize"):
425            have_plural = True
426            next(parser.stream)
427            if parser.stream.current.type != "block_end":
428                token = parser.stream.expect("name")
429                if token.value not in variables:
430                    parser.fail(
431                        f"unknown variable {token.value!r} for pluralization",
432                        token.lineno,
433                        exc=TemplateAssertionError,
434                    )
435                plural_expr = variables[token.value]
436                num_called_num = token.value == "num"
437            parser.stream.expect("block_end")
438            plural_names, plural = self._parse_block(parser, False)
439            next(parser.stream)
440            referenced.update(plural_names)
441        else:
442            next(parser.stream)
443
444        # register free names as simple name expressions
445        for name in referenced:
446            if name not in variables:
447                variables[name] = nodes.Name(name, "load")
448
449        if not have_plural:
450            plural_expr = None
451        elif plural_expr is None:
452            parser.fail("pluralize without variables", lineno)
453
454        if trimmed is None:
455            trimmed = self.environment.policies["ext.i18n.trimmed"]
456        if trimmed:
457            singular = self._trim_whitespace(singular)
458            if plural:
459                plural = self._trim_whitespace(plural)
460
461        node = self._make_node(
462            singular,
463            plural,
464            context,
465            variables,
466            plural_expr,
467            bool(referenced),
468            num_called_num and have_plural,
469        )
470        node.set_lineno(lineno)
471        if plural_expr_assignment is not None:
472            return [plural_expr_assignment, node]
473        else:
474            return node
475
476    def _trim_whitespace(self, string: str, _ws_re: t.Pattern[str] = _ws_re) -> str:
477        return _ws_re.sub(" ", string.strip())
478
479    def _parse_block(
480        self, parser: "Parser", allow_pluralize: bool
481    ) -> t.Tuple[t.List[str], str]:
482        """Parse until the next block tag with a given name."""
483        referenced = []
484        buf = []
485
486        while True:
487            if parser.stream.current.type == "data":
488                buf.append(parser.stream.current.value.replace("%", "%%"))
489                next(parser.stream)
490            elif parser.stream.current.type == "variable_begin":
491                next(parser.stream)
492                name = parser.stream.expect("name").value
493                referenced.append(name)
494                buf.append(f"%({name})s")
495                parser.stream.expect("variable_end")
496            elif parser.stream.current.type == "block_begin":
497                next(parser.stream)
498                block_name = (
499                    parser.stream.current.value
500                    if parser.stream.current.type == "name"
501                    else None
502                )
503                if block_name == "endtrans":
504                    break
505                elif block_name == "pluralize":
506                    if allow_pluralize:
507                        break
508                    parser.fail(
509                        "a translatable section can have only one pluralize section"
510                    )
511                elif block_name == "trans":
512                    parser.fail(
513                        "trans blocks can't be nested; did you mean `endtrans`?"
514                    )
515                parser.fail(
516                    f"control structures in translatable sections are not allowed; "
517                    f"saw `{block_name}`"
518                )
519            elif parser.stream.eos:
520                parser.fail("unclosed translation block")
521            else:
522                raise RuntimeError("internal parser error")
523
524        return referenced, concat(buf)
525
526    def _make_node(
527        self,
528        singular: str,
529        plural: t.Optional[str],
530        context: t.Optional[str],
531        variables: t.Dict[str, nodes.Expr],
532        plural_expr: t.Optional[nodes.Expr],
533        vars_referenced: bool,
534        num_called_num: bool,
535    ) -> nodes.Output:
536        """Generates a useful node from the data provided."""
537        newstyle = self.environment.newstyle_gettext  # type: ignore
538        node: nodes.Expr
539
540        # no variables referenced?  no need to escape for old style
541        # gettext invocations only if there are vars.
542        if not vars_referenced and not newstyle:
543            singular = singular.replace("%%", "%")
544            if plural:
545                plural = plural.replace("%%", "%")
546
547        func_name = "gettext"
548        func_args: t.List[nodes.Expr] = [nodes.Const(singular)]
549
550        if context is not None:
551            func_args.insert(0, nodes.Const(context))
552            func_name = f"p{func_name}"
553
554        if plural_expr is not None:
555            func_name = f"n{func_name}"
556            func_args.extend((nodes.Const(plural), plural_expr))
557
558        node = nodes.Call(nodes.Name(func_name, "load"), func_args, [], None, None)
559
560        # in case newstyle gettext is used, the method is powerful
561        # enough to handle the variable expansion and autoescape
562        # handling itself
563        if newstyle:
564            for key, value in variables.items():
565                # the function adds that later anyways in case num was
566                # called num, so just skip it.
567                if num_called_num and key == "num":
568                    continue
569                node.kwargs.append(nodes.Keyword(key, value))
570
571        # otherwise do that here
572        else:
573            # mark the return value as safe if we are in an
574            # environment with autoescaping turned on
575            node = nodes.MarkSafeIfAutoescape(node)
576            if variables:
577                node = nodes.Mod(
578                    node,
579                    nodes.Dict(
580                        [
581                            nodes.Pair(nodes.Const(key), value)
582                            for key, value in variables.items()
583                        ]
584                    ),
585                )
586        return nodes.Output([node])
587
588
589class ExprStmtExtension(Extension):
590    """Adds a `do` tag to Jinja that works like the print statement just
591    that it doesn't print the return value.
592    """
593
594    tags = {"do"}
595
596    def parse(self, parser: "Parser") -> nodes.ExprStmt:
597        node = nodes.ExprStmt(lineno=next(parser.stream).lineno)
598        node.node = parser.parse_tuple()
599        return node
600
601
602class LoopControlExtension(Extension):
603    """Adds break and continue to the template engine."""
604
605    tags = {"break", "continue"}
606
607    def parse(self, parser: "Parser") -> t.Union[nodes.Break, nodes.Continue]:
608        token = next(parser.stream)
609        if token.value == "break":
610            return nodes.Break(lineno=token.lineno)
611        return nodes.Continue(lineno=token.lineno)
612
613
614class DebugExtension(Extension):
615    """A ``{% debug %}`` tag that dumps the available variables,
616    filters, and tests.
617
618    .. code-block:: html+jinja
619
620        <pre>{% debug %}</pre>
621
622    .. code-block:: text
623
624        {'context': {'cycler': <class 'jinja2.utils.Cycler'>,
625                     ...,
626                     'namespace': <class 'jinja2.utils.Namespace'>},
627         'filters': ['abs', 'attr', 'batch', 'capitalize', 'center', 'count', 'd',
628                     ..., 'urlencode', 'urlize', 'wordcount', 'wordwrap', 'xmlattr'],
629         'tests': ['!=', '<', '<=', '==', '>', '>=', 'callable', 'defined',
630                   ..., 'odd', 'sameas', 'sequence', 'string', 'undefined', 'upper']}
631
632    .. versionadded:: 2.11.0
633    """
634
635    tags = {"debug"}
636
637    def parse(self, parser: "Parser") -> nodes.Output:
638        lineno = parser.stream.expect("name:debug").lineno
639        context = nodes.ContextReference()
640        result = self.call_method("_render", [context], lineno=lineno)
641        return nodes.Output([result], lineno=lineno)
642
643    def _render(self, context: Context) -> str:
644        result = {
645            "context": context.get_all(),
646            "filters": sorted(self.environment.filters.keys()),
647            "tests": sorted(self.environment.tests.keys()),
648        }
649
650        # Set the depth since the intent is to show the top few names.
651        return pprint.pformat(result, depth=3, compact=True)
652
653
654def extract_from_ast(
655    ast: nodes.Template,
656    gettext_functions: t.Sequence[str] = GETTEXT_FUNCTIONS,
657    babel_style: bool = True,
658) -> t.Iterator[
659    t.Tuple[int, str, t.Union[t.Optional[str], t.Tuple[t.Optional[str], ...]]]
660]:
661    """Extract localizable strings from the given template node.  Per
662    default this function returns matches in babel style that means non string
663    parameters as well as keyword arguments are returned as `None`.  This
664    allows Babel to figure out what you really meant if you are using
665    gettext functions that allow keyword arguments for placeholder expansion.
666    If you don't want that behavior set the `babel_style` parameter to `False`
667    which causes only strings to be returned and parameters are always stored
668    in tuples.  As a consequence invalid gettext calls (calls without a single
669    string parameter or string parameters after non-string parameters) are
670    skipped.
671
672    This example explains the behavior:
673
674    >>> from jinja2 import Environment
675    >>> env = Environment()
676    >>> node = env.parse('{{ (_("foo"), _(), ngettext("foo", "bar", 42)) }}')
677    >>> list(extract_from_ast(node))
678    [(1, '_', 'foo'), (1, '_', ()), (1, 'ngettext', ('foo', 'bar', None))]
679    >>> list(extract_from_ast(node, babel_style=False))
680    [(1, '_', ('foo',)), (1, 'ngettext', ('foo', 'bar'))]
681
682    For every string found this function yields a ``(lineno, function,
683    message)`` tuple, where:
684
685    * ``lineno`` is the number of the line on which the string was found,
686    * ``function`` is the name of the ``gettext`` function used (if the
687      string was extracted from embedded Python code), and
688    *   ``message`` is the string, or a tuple of strings for functions
689         with multiple string arguments.
690
691    This extraction function operates on the AST and is because of that unable
692    to extract any comments.  For comment support you have to use the babel
693    extraction interface or extract comments yourself.
694    """
695    out: t.Union[t.Optional[str], t.Tuple[t.Optional[str], ...]]
696
697    for node in ast.find_all(nodes.Call):
698        if (
699            not isinstance(node.node, nodes.Name)
700            or node.node.name not in gettext_functions
701        ):
702            continue
703
704        strings: t.List[t.Optional[str]] = []
705
706        for arg in node.args:
707            if isinstance(arg, nodes.Const) and isinstance(arg.value, str):
708                strings.append(arg.value)
709            else:
710                strings.append(None)
711
712        for _ in node.kwargs:
713            strings.append(None)
714        if node.dyn_args is not None:
715            strings.append(None)
716        if node.dyn_kwargs is not None:
717            strings.append(None)
718
719        if not babel_style:
720            out = tuple(x for x in strings if x is not None)
721
722            if not out:
723                continue
724        else:
725            if len(strings) == 1:
726                out = strings[0]
727            else:
728                out = tuple(strings)
729
730        yield node.lineno, node.node.name, out
731
732
733class _CommentFinder:
734    """Helper class to find comments in a token stream.  Can only
735    find comments for gettext calls forwards.  Once the comment
736    from line 4 is found, a comment for line 1 will not return a
737    usable value.
738    """
739
740    def __init__(
741        self, tokens: t.Sequence[t.Tuple[int, str, str]], comment_tags: t.Sequence[str]
742    ) -> None:
743        self.tokens = tokens
744        self.comment_tags = comment_tags
745        self.offset = 0
746        self.last_lineno = 0
747
748    def find_backwards(self, offset: int) -> t.List[str]:
749        try:
750            for _, token_type, token_value in reversed(
751                self.tokens[self.offset : offset]
752            ):
753                if token_type in ("comment", "linecomment"):
754                    try:
755                        prefix, comment = token_value.split(None, 1)
756                    except ValueError:
757                        continue
758                    if prefix in self.comment_tags:
759                        return [comment.rstrip()]
760            return []
761        finally:
762            self.offset = offset
763
764    def find_comments(self, lineno: int) -> t.List[str]:
765        if not self.comment_tags or self.last_lineno > lineno:
766            return []
767        for idx, (token_lineno, _, _) in enumerate(self.tokens[self.offset :]):
768            if token_lineno > lineno:
769                return self.find_backwards(self.offset + idx)
770        return self.find_backwards(len(self.tokens))
771
772
773def babel_extract(
774    fileobj: t.BinaryIO,
775    keywords: t.Sequence[str],
776    comment_tags: t.Sequence[str],
777    options: t.Dict[str, t.Any],
778) -> t.Iterator[
779    t.Tuple[
780        int, str, t.Union[t.Optional[str], t.Tuple[t.Optional[str], ...]], t.List[str]
781    ]
782]:
783    """Babel extraction method for Jinja templates.
784
785    .. versionchanged:: 2.3
786       Basic support for translation comments was added.  If `comment_tags`
787       is now set to a list of keywords for extraction, the extractor will
788       try to find the best preceding comment that begins with one of the
789       keywords.  For best results, make sure to not have more than one
790       gettext call in one line of code and the matching comment in the
791       same line or the line before.
792
793    .. versionchanged:: 2.5.1
794       The `newstyle_gettext` flag can be set to `True` to enable newstyle
795       gettext calls.
796
797    .. versionchanged:: 2.7
798       A `silent` option can now be provided.  If set to `False` template
799       syntax errors are propagated instead of being ignored.
800
801    :param fileobj: the file-like object the messages should be extracted from
802    :param keywords: a list of keywords (i.e. function names) that should be
803                     recognized as translation functions
804    :param comment_tags: a list of translator tags to search for and include
805                         in the results.
806    :param options: a dictionary of additional options (optional)
807    :return: an iterator over ``(lineno, funcname, message, comments)`` tuples.
808             (comments will be empty currently)
809    """
810    extensions: t.Dict[t.Type[Extension], None] = {}
811
812    for extension_name in options.get("extensions", "").split(","):
813        extension_name = extension_name.strip()
814
815        if not extension_name:
816            continue
817
818        extensions[import_string(extension_name)] = None
819
820    if InternationalizationExtension not in extensions:
821        extensions[InternationalizationExtension] = None
822
823    def getbool(options: t.Mapping[str, str], key: str, default: bool = False) -> bool:
824        return options.get(key, str(default)).lower() in {"1", "on", "yes", "true"}
825
826    silent = getbool(options, "silent", True)
827    environment = Environment(
828        options.get("block_start_string", defaults.BLOCK_START_STRING),
829        options.get("block_end_string", defaults.BLOCK_END_STRING),
830        options.get("variable_start_string", defaults.VARIABLE_START_STRING),
831        options.get("variable_end_string", defaults.VARIABLE_END_STRING),
832        options.get("comment_start_string", defaults.COMMENT_START_STRING),
833        options.get("comment_end_string", defaults.COMMENT_END_STRING),
834        options.get("line_statement_prefix") or defaults.LINE_STATEMENT_PREFIX,
835        options.get("line_comment_prefix") or defaults.LINE_COMMENT_PREFIX,
836        getbool(options, "trim_blocks", defaults.TRIM_BLOCKS),
837        getbool(options, "lstrip_blocks", defaults.LSTRIP_BLOCKS),
838        defaults.NEWLINE_SEQUENCE,
839        getbool(options, "keep_trailing_newline", defaults.KEEP_TRAILING_NEWLINE),
840        tuple(extensions),
841        cache_size=0,
842        auto_reload=False,
843    )
844
845    if getbool(options, "trimmed"):
846        environment.policies["ext.i18n.trimmed"] = True
847    if getbool(options, "newstyle_gettext"):
848        environment.newstyle_gettext = True  # type: ignore
849
850    source = fileobj.read().decode(options.get("encoding", "utf-8"))
851    try:
852        node = environment.parse(source)
853        tokens = list(environment.lex(environment.preprocess(source)))
854    except TemplateSyntaxError:
855        if not silent:
856            raise
857        # skip templates with syntax errors
858        return
859
860    finder = _CommentFinder(tokens, comment_tags)
861    for lineno, func, message in extract_from_ast(node, keywords):
862        yield lineno, func, message, finder.find_comments(lineno)
863
864
865#: nicer import names
866i18n = InternationalizationExtension
867do = ExprStmtExtension
868loopcontrols = LoopControlExtension
869debug = DebugExtension