jinja2.loaders

  1"""API and implementations for loading templates from different data
  2sources.
  3"""
  4import importlib.util
  5import os
  6import posixpath
  7import sys
  8import typing as t
  9import weakref
 10import zipimport
 11from collections import abc
 12from hashlib import sha1
 13from importlib import import_module
 14from types import ModuleType
 15
 16from .exceptions import TemplateNotFound
 17from .utils import internalcode
 18
 19if t.TYPE_CHECKING:
 20    from .environment import Environment
 21    from .environment import Template
 22
 23
 24def split_template_path(template: str) -> t.List[str]:
 25    """Split a path into segments and perform a sanity check.  If it detects
 26    '..' in the path it will raise a `TemplateNotFound` error.
 27    """
 28    pieces = []
 29    for piece in template.split("/"):
 30        if (
 31            os.path.sep in piece
 32            or (os.path.altsep and os.path.altsep in piece)
 33            or piece == os.path.pardir
 34        ):
 35            raise TemplateNotFound(template)
 36        elif piece and piece != ".":
 37            pieces.append(piece)
 38    return pieces
 39
 40
 41class BaseLoader:
 42    """Baseclass for all loaders.  Subclass this and override `get_source` to
 43    implement a custom loading mechanism.  The environment provides a
 44    `get_template` method that calls the loader's `load` method to get the
 45    :class:`Template` object.
 46
 47    A very basic example for a loader that looks up templates on the file
 48    system could look like this::
 49
 50        from jinja2 import BaseLoader, TemplateNotFound
 51        from os.path import join, exists, getmtime
 52
 53        class MyLoader(BaseLoader):
 54
 55            def __init__(self, path):
 56                self.path = path
 57
 58            def get_source(self, environment, template):
 59                path = join(self.path, template)
 60                if not exists(path):
 61                    raise TemplateNotFound(template)
 62                mtime = getmtime(path)
 63                with open(path) as f:
 64                    source = f.read()
 65                return source, path, lambda: mtime == getmtime(path)
 66    """
 67
 68    #: if set to `False` it indicates that the loader cannot provide access
 69    #: to the source of templates.
 70    #:
 71    #: .. versionadded:: 2.4
 72    has_source_access = True
 73
 74    def get_source(
 75        self, environment: "Environment", template: str
 76    ) -> t.Tuple[str, t.Optional[str], t.Optional[t.Callable[[], bool]]]:
 77        """Get the template source, filename and reload helper for a template.
 78        It's passed the environment and template name and has to return a
 79        tuple in the form ``(source, filename, uptodate)`` or raise a
 80        `TemplateNotFound` error if it can't locate the template.
 81
 82        The source part of the returned tuple must be the source of the
 83        template as a string. The filename should be the name of the
 84        file on the filesystem if it was loaded from there, otherwise
 85        ``None``. The filename is used by Python for the tracebacks
 86        if no loader extension is used.
 87
 88        The last item in the tuple is the `uptodate` function.  If auto
 89        reloading is enabled it's always called to check if the template
 90        changed.  No arguments are passed so the function must store the
 91        old state somewhere (for example in a closure).  If it returns `False`
 92        the template will be reloaded.
 93        """
 94        if not self.has_source_access:
 95            raise RuntimeError(
 96                f"{type(self).__name__} cannot provide access to the source"
 97            )
 98        raise TemplateNotFound(template)
 99
100    def list_templates(self) -> t.List[str]:
101        """Iterates over all templates.  If the loader does not support that
102        it should raise a :exc:`TypeError` which is the default behavior.
103        """
104        raise TypeError("this loader cannot iterate over all templates")
105
106    @internalcode
107    def load(
108        self,
109        environment: "Environment",
110        name: str,
111        globals: t.Optional[t.MutableMapping[str, t.Any]] = None,
112    ) -> "Template":
113        """Loads a template.  This method looks up the template in the cache
114        or loads one by calling :meth:`get_source`.  Subclasses should not
115        override this method as loaders working on collections of other
116        loaders (such as :class:`PrefixLoader` or :class:`ChoiceLoader`)
117        will not call this method but `get_source` directly.
118        """
119        code = None
120        if globals is None:
121            globals = {}
122
123        # first we try to get the source for this template together
124        # with the filename and the uptodate function.
125        source, filename, uptodate = self.get_source(environment, name)
126
127        # try to load the code from the bytecode cache if there is a
128        # bytecode cache configured.
129        bcc = environment.bytecode_cache
130        if bcc is not None:
131            bucket = bcc.get_bucket(environment, name, filename, source)
132            code = bucket.code
133
134        # if we don't have code so far (not cached, no longer up to
135        # date) etc. we compile the template
136        if code is None:
137            code = environment.compile(source, name, filename)
138
139        # if the bytecode cache is available and the bucket doesn't
140        # have a code so far, we give the bucket the new code and put
141        # it back to the bytecode cache.
142        if bcc is not None and bucket.code is None:
143            bucket.code = code
144            bcc.set_bucket(bucket)
145
146        return environment.template_class.from_code(
147            environment, code, globals, uptodate
148        )
149
150
151class FileSystemLoader(BaseLoader):
152    """Load templates from a directory in the file system.
153
154    The path can be relative or absolute. Relative paths are relative to
155    the current working directory.
156
157    .. code-block:: python
158
159        loader = FileSystemLoader("templates")
160
161    A list of paths can be given. The directories will be searched in
162    order, stopping at the first matching template.
163
164    .. code-block:: python
165
166        loader = FileSystemLoader(["/override/templates", "/default/templates"])
167
168    :param searchpath: A path, or list of paths, to the directory that
169        contains the templates.
170    :param encoding: Use this encoding to read the text from template
171        files.
172    :param followlinks: Follow symbolic links in the path.
173
174    .. versionchanged:: 2.8
175        Added the ``followlinks`` parameter.
176    """
177
178    def __init__(
179        self,
180        searchpath: t.Union[str, os.PathLike, t.Sequence[t.Union[str, os.PathLike]]],
181        encoding: str = "utf-8",
182        followlinks: bool = False,
183    ) -> None:
184        if not isinstance(searchpath, abc.Iterable) or isinstance(searchpath, str):
185            searchpath = [searchpath]
186
187        self.searchpath = [os.fspath(p) for p in searchpath]
188        self.encoding = encoding
189        self.followlinks = followlinks
190
191    def get_source(
192        self, environment: "Environment", template: str
193    ) -> t.Tuple[str, str, t.Callable[[], bool]]:
194        pieces = split_template_path(template)
195
196        for searchpath in self.searchpath:
197            # Use posixpath even on Windows to avoid "drive:" or UNC
198            # segments breaking out of the search directory.
199            filename = posixpath.join(searchpath, *pieces)
200
201            if os.path.isfile(filename):
202                break
203        else:
204            raise TemplateNotFound(template)
205
206        with open(filename, encoding=self.encoding) as f:
207            contents = f.read()
208
209        mtime = os.path.getmtime(filename)
210
211        def uptodate() -> bool:
212            try:
213                return os.path.getmtime(filename) == mtime
214            except OSError:
215                return False
216
217        # Use normpath to convert Windows altsep to sep.
218        return contents, os.path.normpath(filename), uptodate
219
220    def list_templates(self) -> t.List[str]:
221        found = set()
222        for searchpath in self.searchpath:
223            walk_dir = os.walk(searchpath, followlinks=self.followlinks)
224            for dirpath, _, filenames in walk_dir:
225                for filename in filenames:
226                    template = (
227                        os.path.join(dirpath, filename)[len(searchpath) :]
228                        .strip(os.path.sep)
229                        .replace(os.path.sep, "/")
230                    )
231                    if template[:2] == "./":
232                        template = template[2:]
233                    if template not in found:
234                        found.add(template)
235        return sorted(found)
236
237
238class PackageLoader(BaseLoader):
239    """Load templates from a directory in a Python package.
240
241    :param package_name: Import name of the package that contains the
242        template directory.
243    :param package_path: Directory within the imported package that
244        contains the templates.
245    :param encoding: Encoding of template files.
246
247    The following example looks up templates in the ``pages`` directory
248    within the ``project.ui`` package.
249
250    .. code-block:: python
251
252        loader = PackageLoader("project.ui", "pages")
253
254    Only packages installed as directories (standard pip behavior) or
255    zip/egg files (less common) are supported. The Python API for
256    introspecting data in packages is too limited to support other
257    installation methods the way this loader requires.
258
259    There is limited support for :pep:`420` namespace packages. The
260    template directory is assumed to only be in one namespace
261    contributor. Zip files contributing to a namespace are not
262    supported.
263
264    .. versionchanged:: 3.0
265        No longer uses ``setuptools`` as a dependency.
266
267    .. versionchanged:: 3.0
268        Limited PEP 420 namespace package support.
269    """
270
271    def __init__(
272        self,
273        package_name: str,
274        package_path: "str" = "templates",
275        encoding: str = "utf-8",
276    ) -> None:
277        package_path = os.path.normpath(package_path).rstrip(os.path.sep)
278
279        # normpath preserves ".", which isn't valid in zip paths.
280        if package_path == os.path.curdir:
281            package_path = ""
282        elif package_path[:2] == os.path.curdir + os.path.sep:
283            package_path = package_path[2:]
284
285        self.package_path = package_path
286        self.package_name = package_name
287        self.encoding = encoding
288
289        # Make sure the package exists. This also makes namespace
290        # packages work, otherwise get_loader returns None.
291        import_module(package_name)
292        spec = importlib.util.find_spec(package_name)
293        assert spec is not None, "An import spec was not found for the package."
294        loader = spec.loader
295        assert loader is not None, "A loader was not found for the package."
296        self._loader = loader
297        self._archive = None
298        template_root = None
299
300        if isinstance(loader, zipimport.zipimporter):
301            self._archive = loader.archive
302            pkgdir = next(iter(spec.submodule_search_locations))  # type: ignore
303            template_root = os.path.join(pkgdir, package_path).rstrip(os.path.sep)
304        else:
305            roots: t.List[str] = []
306
307            # One element for regular packages, multiple for namespace
308            # packages, or None for single module file.
309            if spec.submodule_search_locations:
310                roots.extend(spec.submodule_search_locations)
311            # A single module file, use the parent directory instead.
312            elif spec.origin is not None:
313                roots.append(os.path.dirname(spec.origin))
314
315            for root in roots:
316                root = os.path.join(root, package_path)
317
318                if os.path.isdir(root):
319                    template_root = root
320                    break
321
322        if template_root is None:
323            raise ValueError(
324                f"The {package_name!r} package was not installed in a"
325                " way that PackageLoader understands."
326            )
327
328        self._template_root = template_root
329
330    def get_source(
331        self, environment: "Environment", template: str
332    ) -> t.Tuple[str, str, t.Optional[t.Callable[[], bool]]]:
333        # Use posixpath even on Windows to avoid "drive:" or UNC
334        # segments breaking out of the search directory. Use normpath to
335        # convert Windows altsep to sep.
336        p = os.path.normpath(
337            posixpath.join(self._template_root, *split_template_path(template))
338        )
339        up_to_date: t.Optional[t.Callable[[], bool]]
340
341        if self._archive is None:
342            # Package is a directory.
343            if not os.path.isfile(p):
344                raise TemplateNotFound(template)
345
346            with open(p, "rb") as f:
347                source = f.read()
348
349            mtime = os.path.getmtime(p)
350
351            def up_to_date() -> bool:
352                return os.path.isfile(p) and os.path.getmtime(p) == mtime
353
354        else:
355            # Package is a zip file.
356            try:
357                source = self._loader.get_data(p)  # type: ignore
358            except OSError as e:
359                raise TemplateNotFound(template) from e
360
361            # Could use the zip's mtime for all template mtimes, but
362            # would need to safely reload the module if it's out of
363            # date, so just report it as always current.
364            up_to_date = None
365
366        return source.decode(self.encoding), p, up_to_date
367
368    def list_templates(self) -> t.List[str]:
369        results: t.List[str] = []
370
371        if self._archive is None:
372            # Package is a directory.
373            offset = len(self._template_root)
374
375            for dirpath, _, filenames in os.walk(self._template_root):
376                dirpath = dirpath[offset:].lstrip(os.path.sep)
377                results.extend(
378                    os.path.join(dirpath, name).replace(os.path.sep, "/")
379                    for name in filenames
380                )
381        else:
382            if not hasattr(self._loader, "_files"):
383                raise TypeError(
384                    "This zip import does not have the required"
385                    " metadata to list templates."
386                )
387
388            # Package is a zip file.
389            prefix = (
390                self._template_root[len(self._archive) :].lstrip(os.path.sep)
391                + os.path.sep
392            )
393            offset = len(prefix)
394
395            for name in self._loader._files.keys():
396                # Find names under the templates directory that aren't directories.
397                if name.startswith(prefix) and name[-1] != os.path.sep:
398                    results.append(name[offset:].replace(os.path.sep, "/"))
399
400        results.sort()
401        return results
402
403
404class DictLoader(BaseLoader):
405    """Loads a template from a Python dict mapping template names to
406    template source.  This loader is useful for unittesting:
407
408    >>> loader = DictLoader({'index.html': 'source here'})
409
410    Because auto reloading is rarely useful this is disabled per default.
411    """
412
413    def __init__(self, mapping: t.Mapping[str, str]) -> None:
414        self.mapping = mapping
415
416    def get_source(
417        self, environment: "Environment", template: str
418    ) -> t.Tuple[str, None, t.Callable[[], bool]]:
419        if template in self.mapping:
420            source = self.mapping[template]
421            return source, None, lambda: source == self.mapping.get(template)
422        raise TemplateNotFound(template)
423
424    def list_templates(self) -> t.List[str]:
425        return sorted(self.mapping)
426
427
428class FunctionLoader(BaseLoader):
429    """A loader that is passed a function which does the loading.  The
430    function receives the name of the template and has to return either
431    a string with the template source, a tuple in the form ``(source,
432    filename, uptodatefunc)`` or `None` if the template does not exist.
433
434    >>> def load_template(name):
435    ...     if name == 'index.html':
436    ...         return '...'
437    ...
438    >>> loader = FunctionLoader(load_template)
439
440    The `uptodatefunc` is a function that is called if autoreload is enabled
441    and has to return `True` if the template is still up to date.  For more
442    details have a look at :meth:`BaseLoader.get_source` which has the same
443    return value.
444    """
445
446    def __init__(
447        self,
448        load_func: t.Callable[
449            [str],
450            t.Optional[
451                t.Union[
452                    str, t.Tuple[str, t.Optional[str], t.Optional[t.Callable[[], bool]]]
453                ]
454            ],
455        ],
456    ) -> None:
457        self.load_func = load_func
458
459    def get_source(
460        self, environment: "Environment", template: str
461    ) -> t.Tuple[str, t.Optional[str], t.Optional[t.Callable[[], bool]]]:
462        rv = self.load_func(template)
463
464        if rv is None:
465            raise TemplateNotFound(template)
466
467        if isinstance(rv, str):
468            return rv, None, None
469
470        return rv
471
472
473class PrefixLoader(BaseLoader):
474    """A loader that is passed a dict of loaders where each loader is bound
475    to a prefix.  The prefix is delimited from the template by a slash per
476    default, which can be changed by setting the `delimiter` argument to
477    something else::
478
479        loader = PrefixLoader({
480            'app1':     PackageLoader('mypackage.app1'),
481            'app2':     PackageLoader('mypackage.app2')
482        })
483
484    By loading ``'app1/index.html'`` the file from the app1 package is loaded,
485    by loading ``'app2/index.html'`` the file from the second.
486    """
487
488    def __init__(
489        self, mapping: t.Mapping[str, BaseLoader], delimiter: str = "/"
490    ) -> None:
491        self.mapping = mapping
492        self.delimiter = delimiter
493
494    def get_loader(self, template: str) -> t.Tuple[BaseLoader, str]:
495        try:
496            prefix, name = template.split(self.delimiter, 1)
497            loader = self.mapping[prefix]
498        except (ValueError, KeyError) as e:
499            raise TemplateNotFound(template) from e
500        return loader, name
501
502    def get_source(
503        self, environment: "Environment", template: str
504    ) -> t.Tuple[str, t.Optional[str], t.Optional[t.Callable[[], bool]]]:
505        loader, name = self.get_loader(template)
506        try:
507            return loader.get_source(environment, name)
508        except TemplateNotFound as e:
509            # re-raise the exception with the correct filename here.
510            # (the one that includes the prefix)
511            raise TemplateNotFound(template) from e
512
513    @internalcode
514    def load(
515        self,
516        environment: "Environment",
517        name: str,
518        globals: t.Optional[t.MutableMapping[str, t.Any]] = None,
519    ) -> "Template":
520        loader, local_name = self.get_loader(name)
521        try:
522            return loader.load(environment, local_name, globals)
523        except TemplateNotFound as e:
524            # re-raise the exception with the correct filename here.
525            # (the one that includes the prefix)
526            raise TemplateNotFound(name) from e
527
528    def list_templates(self) -> t.List[str]:
529        result = []
530        for prefix, loader in self.mapping.items():
531            for template in loader.list_templates():
532                result.append(prefix + self.delimiter + template)
533        return result
534
535
536class ChoiceLoader(BaseLoader):
537    """This loader works like the `PrefixLoader` just that no prefix is
538    specified.  If a template could not be found by one loader the next one
539    is tried.
540
541    >>> loader = ChoiceLoader([
542    ...     FileSystemLoader('/path/to/user/templates'),
543    ...     FileSystemLoader('/path/to/system/templates')
544    ... ])
545
546    This is useful if you want to allow users to override builtin templates
547    from a different location.
548    """
549
550    def __init__(self, loaders: t.Sequence[BaseLoader]) -> None:
551        self.loaders = loaders
552
553    def get_source(
554        self, environment: "Environment", template: str
555    ) -> t.Tuple[str, t.Optional[str], t.Optional[t.Callable[[], bool]]]:
556        for loader in self.loaders:
557            try:
558                return loader.get_source(environment, template)
559            except TemplateNotFound:
560                pass
561        raise TemplateNotFound(template)
562
563    @internalcode
564    def load(
565        self,
566        environment: "Environment",
567        name: str,
568        globals: t.Optional[t.MutableMapping[str, t.Any]] = None,
569    ) -> "Template":
570        for loader in self.loaders:
571            try:
572                return loader.load(environment, name, globals)
573            except TemplateNotFound:
574                pass
575        raise TemplateNotFound(name)
576
577    def list_templates(self) -> t.List[str]:
578        found = set()
579        for loader in self.loaders:
580            found.update(loader.list_templates())
581        return sorted(found)
582
583
584class _TemplateModule(ModuleType):
585    """Like a normal module but with support for weak references"""
586
587
588class ModuleLoader(BaseLoader):
589    """This loader loads templates from precompiled templates.
590
591    Example usage:
592
593    >>> loader = ChoiceLoader([
594    ...     ModuleLoader('/path/to/compiled/templates'),
595    ...     FileSystemLoader('/path/to/templates')
596    ... ])
597
598    Templates can be precompiled with :meth:`Environment.compile_templates`.
599    """
600
601    has_source_access = False
602
603    def __init__(
604        self, path: t.Union[str, os.PathLike, t.Sequence[t.Union[str, os.PathLike]]]
605    ) -> None:
606        package_name = f"_jinja2_module_templates_{id(self):x}"
607
608        # create a fake module that looks for the templates in the
609        # path given.
610        mod = _TemplateModule(package_name)
611
612        if not isinstance(path, abc.Iterable) or isinstance(path, str):
613            path = [path]
614
615        mod.__path__ = [os.fspath(p) for p in path]
616
617        sys.modules[package_name] = weakref.proxy(
618            mod, lambda x: sys.modules.pop(package_name, None)
619        )
620
621        # the only strong reference, the sys.modules entry is weak
622        # so that the garbage collector can remove it once the
623        # loader that created it goes out of business.
624        self.module = mod
625        self.package_name = package_name
626
627    @staticmethod
628    def get_template_key(name: str) -> str:
629        return "tmpl_" + sha1(name.encode("utf-8")).hexdigest()
630
631    @staticmethod
632    def get_module_filename(name: str) -> str:
633        return ModuleLoader.get_template_key(name) + ".py"
634
635    @internalcode
636    def load(
637        self,
638        environment: "Environment",
639        name: str,
640        globals: t.Optional[t.MutableMapping[str, t.Any]] = None,
641    ) -> "Template":
642        key = self.get_template_key(name)
643        module = f"{self.package_name}.{key}"
644        mod = getattr(self.module, module, None)
645
646        if mod is None:
647            try:
648                mod = __import__(module, None, None, ["root"])
649            except ImportError as e:
650                raise TemplateNotFound(name) from e
651
652            # remove the entry from sys.modules, we only want the attribute
653            # on the module object we have stored on the loader.
654            sys.modules.pop(module, None)
655
656        if globals is None:
657            globals = {}
658
659        return environment.template_class.from_module_dict(
660            environment, mod.__dict__, globals
661        )