jinja2.loaders

  1"""API and implementations for loading templates from different data
  2sources.
  3"""
  4
  5import importlib.util
  6import os
  7import posixpath
  8import sys
  9import typing as t
 10import weakref
 11import zipimport
 12from collections import abc
 13from hashlib import sha1
 14from importlib import import_module
 15from types import ModuleType
 16
 17from .exceptions import TemplateNotFound
 18from .utils import internalcode
 19
 20if t.TYPE_CHECKING:
 21    from .environment import Environment
 22    from .environment import Template
 23
 24
 25def split_template_path(template: str) -> t.List[str]:
 26    """Split a path into segments and perform a sanity check.  If it detects
 27    '..' in the path it will raise a `TemplateNotFound` error.
 28    """
 29    pieces = []
 30    for piece in template.split("/"):
 31        if (
 32            os.path.sep in piece
 33            or (os.path.altsep and os.path.altsep in piece)
 34            or piece == os.path.pardir
 35        ):
 36            raise TemplateNotFound(template)
 37        elif piece and piece != ".":
 38            pieces.append(piece)
 39    return pieces
 40
 41
 42class BaseLoader:
 43    """Baseclass for all loaders.  Subclass this and override `get_source` to
 44    implement a custom loading mechanism.  The environment provides a
 45    `get_template` method that calls the loader's `load` method to get the
 46    :class:`Template` object.
 47
 48    A very basic example for a loader that looks up templates on the file
 49    system could look like this::
 50
 51        from jinja2 import BaseLoader, TemplateNotFound
 52        from os.path import join, exists, getmtime
 53
 54        class MyLoader(BaseLoader):
 55
 56            def __init__(self, path):
 57                self.path = path
 58
 59            def get_source(self, environment, template):
 60                path = join(self.path, template)
 61                if not exists(path):
 62                    raise TemplateNotFound(template)
 63                mtime = getmtime(path)
 64                with open(path) as f:
 65                    source = f.read()
 66                return source, path, lambda: mtime == getmtime(path)
 67    """
 68
 69    #: if set to `False` it indicates that the loader cannot provide access
 70    #: to the source of templates.
 71    #:
 72    #: .. versionadded:: 2.4
 73    has_source_access = True
 74
 75    def get_source(
 76        self, environment: "Environment", template: str
 77    ) -> t.Tuple[str, t.Optional[str], t.Optional[t.Callable[[], bool]]]:
 78        """Get the template source, filename and reload helper for a template.
 79        It's passed the environment and template name and has to return a
 80        tuple in the form ``(source, filename, uptodate)`` or raise a
 81        `TemplateNotFound` error if it can't locate the template.
 82
 83        The source part of the returned tuple must be the source of the
 84        template as a string. The filename should be the name of the
 85        file on the filesystem if it was loaded from there, otherwise
 86        ``None``. The filename is used by Python for the tracebacks
 87        if no loader extension is used.
 88
 89        The last item in the tuple is the `uptodate` function.  If auto
 90        reloading is enabled it's always called to check if the template
 91        changed.  No arguments are passed so the function must store the
 92        old state somewhere (for example in a closure).  If it returns `False`
 93        the template will be reloaded.
 94        """
 95        if not self.has_source_access:
 96            raise RuntimeError(
 97                f"{type(self).__name__} cannot provide access to the source"
 98            )
 99        raise TemplateNotFound(template)
100
101    def list_templates(self) -> t.List[str]:
102        """Iterates over all templates.  If the loader does not support that
103        it should raise a :exc:`TypeError` which is the default behavior.
104        """
105        raise TypeError("this loader cannot iterate over all templates")
106
107    @internalcode
108    def load(
109        self,
110        environment: "Environment",
111        name: str,
112        globals: t.Optional[t.MutableMapping[str, t.Any]] = None,
113    ) -> "Template":
114        """Loads a template.  This method looks up the template in the cache
115        or loads one by calling :meth:`get_source`.  Subclasses should not
116        override this method as loaders working on collections of other
117        loaders (such as :class:`PrefixLoader` or :class:`ChoiceLoader`)
118        will not call this method but `get_source` directly.
119        """
120        code = None
121        if globals is None:
122            globals = {}
123
124        # first we try to get the source for this template together
125        # with the filename and the uptodate function.
126        source, filename, uptodate = self.get_source(environment, name)
127
128        # try to load the code from the bytecode cache if there is a
129        # bytecode cache configured.
130        bcc = environment.bytecode_cache
131        if bcc is not None:
132            bucket = bcc.get_bucket(environment, name, filename, source)
133            code = bucket.code
134
135        # if we don't have code so far (not cached, no longer up to
136        # date) etc. we compile the template
137        if code is None:
138            code = environment.compile(source, name, filename)
139
140        # if the bytecode cache is available and the bucket doesn't
141        # have a code so far, we give the bucket the new code and put
142        # it back to the bytecode cache.
143        if bcc is not None and bucket.code is None:
144            bucket.code = code
145            bcc.set_bucket(bucket)
146
147        return environment.template_class.from_code(
148            environment, code, globals, uptodate
149        )
150
151
152class FileSystemLoader(BaseLoader):
153    """Load templates from a directory in the file system.
154
155    The path can be relative or absolute. Relative paths are relative to
156    the current working directory.
157
158    .. code-block:: python
159
160        loader = FileSystemLoader("templates")
161
162    A list of paths can be given. The directories will be searched in
163    order, stopping at the first matching template.
164
165    .. code-block:: python
166
167        loader = FileSystemLoader(["/override/templates", "/default/templates"])
168
169    :param searchpath: A path, or list of paths, to the directory that
170        contains the templates.
171    :param encoding: Use this encoding to read the text from template
172        files.
173    :param followlinks: Follow symbolic links in the path.
174
175    .. versionchanged:: 2.8
176        Added the ``followlinks`` parameter.
177    """
178
179    def __init__(
180        self,
181        searchpath: t.Union[
182            str, "os.PathLike[str]", t.Sequence[t.Union[str, "os.PathLike[str]"]]
183        ],
184        encoding: str = "utf-8",
185        followlinks: bool = False,
186    ) -> None:
187        if not isinstance(searchpath, abc.Iterable) or isinstance(searchpath, str):
188            searchpath = [searchpath]
189
190        self.searchpath = [os.fspath(p) for p in searchpath]
191        self.encoding = encoding
192        self.followlinks = followlinks
193
194    def get_source(
195        self, environment: "Environment", template: str
196    ) -> t.Tuple[str, str, t.Callable[[], bool]]:
197        pieces = split_template_path(template)
198
199        for searchpath in self.searchpath:
200            # Use posixpath even on Windows to avoid "drive:" or UNC
201            # segments breaking out of the search directory.
202            filename = posixpath.join(searchpath, *pieces)
203
204            if os.path.isfile(filename):
205                break
206        else:
207            raise TemplateNotFound(template)
208
209        with open(filename, encoding=self.encoding) as f:
210            contents = f.read()
211
212        mtime = os.path.getmtime(filename)
213
214        def uptodate() -> bool:
215            try:
216                return os.path.getmtime(filename) == mtime
217            except OSError:
218                return False
219
220        # Use normpath to convert Windows altsep to sep.
221        return contents, os.path.normpath(filename), uptodate
222
223    def list_templates(self) -> t.List[str]:
224        found = set()
225        for searchpath in self.searchpath:
226            walk_dir = os.walk(searchpath, followlinks=self.followlinks)
227            for dirpath, _, filenames in walk_dir:
228                for filename in filenames:
229                    template = (
230                        os.path.join(dirpath, filename)[len(searchpath) :]
231                        .strip(os.path.sep)
232                        .replace(os.path.sep, "/")
233                    )
234                    if template[:2] == "./":
235                        template = template[2:]
236                    if template not in found:
237                        found.add(template)
238        return sorted(found)
239
240
241class PackageLoader(BaseLoader):
242    """Load templates from a directory in a Python package.
243
244    :param package_name: Import name of the package that contains the
245        template directory.
246    :param package_path: Directory within the imported package that
247        contains the templates.
248    :param encoding: Encoding of template files.
249
250    The following example looks up templates in the ``pages`` directory
251    within the ``project.ui`` package.
252
253    .. code-block:: python
254
255        loader = PackageLoader("project.ui", "pages")
256
257    Only packages installed as directories (standard pip behavior) or
258    zip/egg files (less common) are supported. The Python API for
259    introspecting data in packages is too limited to support other
260    installation methods the way this loader requires.
261
262    There is limited support for :pep:`420` namespace packages. The
263    template directory is assumed to only be in one namespace
264    contributor. Zip files contributing to a namespace are not
265    supported.
266
267    .. versionchanged:: 3.0
268        No longer uses ``setuptools`` as a dependency.
269
270    .. versionchanged:: 3.0
271        Limited PEP 420 namespace package support.
272    """
273
274    def __init__(
275        self,
276        package_name: str,
277        package_path: "str" = "templates",
278        encoding: str = "utf-8",
279    ) -> None:
280        package_path = os.path.normpath(package_path).rstrip(os.path.sep)
281
282        # normpath preserves ".", which isn't valid in zip paths.
283        if package_path == os.path.curdir:
284            package_path = ""
285        elif package_path[:2] == os.path.curdir + os.path.sep:
286            package_path = package_path[2:]
287
288        self.package_path = package_path
289        self.package_name = package_name
290        self.encoding = encoding
291
292        # Make sure the package exists. This also makes namespace
293        # packages work, otherwise get_loader returns None.
294        import_module(package_name)
295        spec = importlib.util.find_spec(package_name)
296        assert spec is not None, "An import spec was not found for the package."
297        loader = spec.loader
298        assert loader is not None, "A loader was not found for the package."
299        self._loader = loader
300        self._archive = None
301        template_root = None
302
303        if isinstance(loader, zipimport.zipimporter):
304            self._archive = loader.archive
305            pkgdir = next(iter(spec.submodule_search_locations))  # type: ignore
306            template_root = os.path.join(pkgdir, package_path).rstrip(os.path.sep)
307        else:
308            roots: t.List[str] = []
309
310            # One element for regular packages, multiple for namespace
311            # packages, or None for single module file.
312            if spec.submodule_search_locations:
313                roots.extend(spec.submodule_search_locations)
314            # A single module file, use the parent directory instead.
315            elif spec.origin is not None:
316                roots.append(os.path.dirname(spec.origin))
317
318            for root in roots:
319                root = os.path.join(root, package_path)
320
321                if os.path.isdir(root):
322                    template_root = root
323                    break
324
325        if template_root is None:
326            raise ValueError(
327                f"The {package_name!r} package was not installed in a"
328                " way that PackageLoader understands."
329            )
330
331        self._template_root = template_root
332
333    def get_source(
334        self, environment: "Environment", template: str
335    ) -> t.Tuple[str, str, t.Optional[t.Callable[[], bool]]]:
336        # Use posixpath even on Windows to avoid "drive:" or UNC
337        # segments breaking out of the search directory. Use normpath to
338        # convert Windows altsep to sep.
339        p = os.path.normpath(
340            posixpath.join(self._template_root, *split_template_path(template))
341        )
342        up_to_date: t.Optional[t.Callable[[], bool]]
343
344        if self._archive is None:
345            # Package is a directory.
346            if not os.path.isfile(p):
347                raise TemplateNotFound(template)
348
349            with open(p, "rb") as f:
350                source = f.read()
351
352            mtime = os.path.getmtime(p)
353
354            def up_to_date() -> bool:
355                return os.path.isfile(p) and os.path.getmtime(p) == mtime
356
357        else:
358            # Package is a zip file.
359            try:
360                source = self._loader.get_data(p)  # type: ignore
361            except OSError as e:
362                raise TemplateNotFound(template) from e
363
364            # Could use the zip's mtime for all template mtimes, but
365            # would need to safely reload the module if it's out of
366            # date, so just report it as always current.
367            up_to_date = None
368
369        return source.decode(self.encoding), p, up_to_date
370
371    def list_templates(self) -> t.List[str]:
372        results: t.List[str] = []
373
374        if self._archive is None:
375            # Package is a directory.
376            offset = len(self._template_root)
377
378            for dirpath, _, filenames in os.walk(self._template_root):
379                dirpath = dirpath[offset:].lstrip(os.path.sep)
380                results.extend(
381                    os.path.join(dirpath, name).replace(os.path.sep, "/")
382                    for name in filenames
383                )
384        else:
385            if not hasattr(self._loader, "_files"):
386                raise TypeError(
387                    "This zip import does not have the required"
388                    " metadata to list templates."
389                )
390
391            # Package is a zip file.
392            prefix = (
393                self._template_root[len(self._archive) :].lstrip(os.path.sep)
394                + os.path.sep
395            )
396            offset = len(prefix)
397
398            for name in self._loader._files.keys():
399                # Find names under the templates directory that aren't directories.
400                if name.startswith(prefix) and name[-1] != os.path.sep:
401                    results.append(name[offset:].replace(os.path.sep, "/"))
402
403        results.sort()
404        return results
405
406
407class DictLoader(BaseLoader):
408    """Loads a template from a Python dict mapping template names to
409    template source.  This loader is useful for unittesting:
410
411    >>> loader = DictLoader({'index.html': 'source here'})
412
413    Because auto reloading is rarely useful this is disabled per default.
414    """
415
416    def __init__(self, mapping: t.Mapping[str, str]) -> None:
417        self.mapping = mapping
418
419    def get_source(
420        self, environment: "Environment", template: str
421    ) -> t.Tuple[str, None, t.Callable[[], bool]]:
422        if template in self.mapping:
423            source = self.mapping[template]
424            return source, None, lambda: source == self.mapping.get(template)
425        raise TemplateNotFound(template)
426
427    def list_templates(self) -> t.List[str]:
428        return sorted(self.mapping)
429
430
431class FunctionLoader(BaseLoader):
432    """A loader that is passed a function which does the loading.  The
433    function receives the name of the template and has to return either
434    a string with the template source, a tuple in the form ``(source,
435    filename, uptodatefunc)`` or `None` if the template does not exist.
436
437    >>> def load_template(name):
438    ...     if name == 'index.html':
439    ...         return '...'
440    ...
441    >>> loader = FunctionLoader(load_template)
442
443    The `uptodatefunc` is a function that is called if autoreload is enabled
444    and has to return `True` if the template is still up to date.  For more
445    details have a look at :meth:`BaseLoader.get_source` which has the same
446    return value.
447    """
448
449    def __init__(
450        self,
451        load_func: t.Callable[
452            [str],
453            t.Optional[
454                t.Union[
455                    str, t.Tuple[str, t.Optional[str], t.Optional[t.Callable[[], bool]]]
456                ]
457            ],
458        ],
459    ) -> None:
460        self.load_func = load_func
461
462    def get_source(
463        self, environment: "Environment", template: str
464    ) -> t.Tuple[str, t.Optional[str], t.Optional[t.Callable[[], bool]]]:
465        rv = self.load_func(template)
466
467        if rv is None:
468            raise TemplateNotFound(template)
469
470        if isinstance(rv, str):
471            return rv, None, None
472
473        return rv
474
475
476class PrefixLoader(BaseLoader):
477    """A loader that is passed a dict of loaders where each loader is bound
478    to a prefix.  The prefix is delimited from the template by a slash per
479    default, which can be changed by setting the `delimiter` argument to
480    something else::
481
482        loader = PrefixLoader({
483            'app1':     PackageLoader('mypackage.app1'),
484            'app2':     PackageLoader('mypackage.app2')
485        })
486
487    By loading ``'app1/index.html'`` the file from the app1 package is loaded,
488    by loading ``'app2/index.html'`` the file from the second.
489    """
490
491    def __init__(
492        self, mapping: t.Mapping[str, BaseLoader], delimiter: str = "/"
493    ) -> None:
494        self.mapping = mapping
495        self.delimiter = delimiter
496
497    def get_loader(self, template: str) -> t.Tuple[BaseLoader, str]:
498        try:
499            prefix, name = template.split(self.delimiter, 1)
500            loader = self.mapping[prefix]
501        except (ValueError, KeyError) as e:
502            raise TemplateNotFound(template) from e
503        return loader, name
504
505    def get_source(
506        self, environment: "Environment", template: str
507    ) -> t.Tuple[str, t.Optional[str], t.Optional[t.Callable[[], bool]]]:
508        loader, name = self.get_loader(template)
509        try:
510            return loader.get_source(environment, name)
511        except TemplateNotFound as e:
512            # re-raise the exception with the correct filename here.
513            # (the one that includes the prefix)
514            raise TemplateNotFound(template) from e
515
516    @internalcode
517    def load(
518        self,
519        environment: "Environment",
520        name: str,
521        globals: t.Optional[t.MutableMapping[str, t.Any]] = None,
522    ) -> "Template":
523        loader, local_name = self.get_loader(name)
524        try:
525            return loader.load(environment, local_name, globals)
526        except TemplateNotFound as e:
527            # re-raise the exception with the correct filename here.
528            # (the one that includes the prefix)
529            raise TemplateNotFound(name) from e
530
531    def list_templates(self) -> t.List[str]:
532        result = []
533        for prefix, loader in self.mapping.items():
534            for template in loader.list_templates():
535                result.append(prefix + self.delimiter + template)
536        return result
537
538
539class ChoiceLoader(BaseLoader):
540    """This loader works like the `PrefixLoader` just that no prefix is
541    specified.  If a template could not be found by one loader the next one
542    is tried.
543
544    >>> loader = ChoiceLoader([
545    ...     FileSystemLoader('/path/to/user/templates'),
546    ...     FileSystemLoader('/path/to/system/templates')
547    ... ])
548
549    This is useful if you want to allow users to override builtin templates
550    from a different location.
551    """
552
553    def __init__(self, loaders: t.Sequence[BaseLoader]) -> None:
554        self.loaders = loaders
555
556    def get_source(
557        self, environment: "Environment", template: str
558    ) -> t.Tuple[str, t.Optional[str], t.Optional[t.Callable[[], bool]]]:
559        for loader in self.loaders:
560            try:
561                return loader.get_source(environment, template)
562            except TemplateNotFound:
563                pass
564        raise TemplateNotFound(template)
565
566    @internalcode
567    def load(
568        self,
569        environment: "Environment",
570        name: str,
571        globals: t.Optional[t.MutableMapping[str, t.Any]] = None,
572    ) -> "Template":
573        for loader in self.loaders:
574            try:
575                return loader.load(environment, name, globals)
576            except TemplateNotFound:
577                pass
578        raise TemplateNotFound(name)
579
580    def list_templates(self) -> t.List[str]:
581        found = set()
582        for loader in self.loaders:
583            found.update(loader.list_templates())
584        return sorted(found)
585
586
587class _TemplateModule(ModuleType):
588    """Like a normal module but with support for weak references"""
589
590
591class ModuleLoader(BaseLoader):
592    """This loader loads templates from precompiled templates.
593
594    Example usage:
595
596    >>> loader = ChoiceLoader([
597    ...     ModuleLoader('/path/to/compiled/templates'),
598    ...     FileSystemLoader('/path/to/templates')
599    ... ])
600
601    Templates can be precompiled with :meth:`Environment.compile_templates`.
602    """
603
604    has_source_access = False
605
606    def __init__(
607        self,
608        path: t.Union[
609            str, "os.PathLike[str]", t.Sequence[t.Union[str, "os.PathLike[str]"]]
610        ],
611    ) -> None:
612        package_name = f"_jinja2_module_templates_{id(self):x}"
613
614        # create a fake module that looks for the templates in the
615        # path given.
616        mod = _TemplateModule(package_name)
617
618        if not isinstance(path, abc.Iterable) or isinstance(path, str):
619            path = [path]
620
621        mod.__path__ = [os.fspath(p) for p in path]
622
623        sys.modules[package_name] = weakref.proxy(
624            mod, lambda x: sys.modules.pop(package_name, None)
625        )
626
627        # the only strong reference, the sys.modules entry is weak
628        # so that the garbage collector can remove it once the
629        # loader that created it goes out of business.
630        self.module = mod
631        self.package_name = package_name
632
633    @staticmethod
634    def get_template_key(name: str) -> str:
635        return "tmpl_" + sha1(name.encode("utf-8")).hexdigest()
636
637    @staticmethod
638    def get_module_filename(name: str) -> str:
639        return ModuleLoader.get_template_key(name) + ".py"
640
641    @internalcode
642    def load(
643        self,
644        environment: "Environment",
645        name: str,
646        globals: t.Optional[t.MutableMapping[str, t.Any]] = None,
647    ) -> "Template":
648        key = self.get_template_key(name)
649        module = f"{self.package_name}.{key}"
650        mod = getattr(self.module, module, None)
651
652        if mod is None:
653            try:
654                mod = __import__(module, None, None, ["root"])
655            except ImportError as e:
656                raise TemplateNotFound(name) from e
657
658            # remove the entry from sys.modules, we only want the attribute
659            # on the module object we have stored on the loader.
660            sys.modules.pop(module, None)
661
662        if globals is None:
663            globals = {}
664
665        return environment.template_class.from_module_dict(
666            environment, mod.__dict__, globals
667        )