logging.config

   1# Copyright 2001-2023 by Vinay Sajip. All Rights Reserved.
   2#
   3# Permission to use, copy, modify, and distribute this software and its
   4# documentation for any purpose and without fee is hereby granted,
   5# provided that the above copyright notice appear in all copies and that
   6# both that copyright notice and this permission notice appear in
   7# supporting documentation, and that the name of Vinay Sajip
   8# not be used in advertising or publicity pertaining to distribution
   9# of the software without specific, written prior permission.
  10# VINAY SAJIP DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE, INCLUDING
  11# ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL
  12# VINAY SAJIP BE LIABLE FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR
  13# ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER
  14# IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT
  15# OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
  16
  17"""
  18Configuration functions for the logging package for Python. The core package
  19is based on PEP 282 and comments thereto in comp.lang.python, and influenced
  20by Apache's log4j system.
  21
  22Copyright (C) 2001-2022 Vinay Sajip. All Rights Reserved.
  23
  24To use, simply 'import logging' and log away!
  25"""
  26
  27import errno
  28import functools
  29import io
  30import logging
  31import logging.handlers
  32import os
  33import queue
  34import re
  35import struct
  36import threading
  37import traceback
  38
  39from socketserver import ThreadingTCPServer, StreamRequestHandler
  40
  41
  42DEFAULT_LOGGING_CONFIG_PORT = 9030
  43
  44RESET_ERROR = errno.ECONNRESET
  45
  46#
  47#   The following code implements a socket listener for on-the-fly
  48#   reconfiguration of logging.
  49#
  50#   _listener holds the server object doing the listening
  51_listener = None
  52
  53def fileConfig(fname, defaults=None, disable_existing_loggers=True, encoding=None):
  54    """
  55    Read the logging configuration from a ConfigParser-format file.
  56
  57    This can be called several times from an application, allowing an end user
  58    the ability to select from various pre-canned configurations (if the
  59    developer provides a mechanism to present the choices and load the chosen
  60    configuration).
  61    """
  62    import configparser
  63
  64    if isinstance(fname, str):
  65        if not os.path.exists(fname):
  66            raise FileNotFoundError(f"{fname} doesn't exist")
  67        elif not os.path.getsize(fname):
  68            raise RuntimeError(f'{fname} is an empty file')
  69
  70    if isinstance(fname, configparser.RawConfigParser):
  71        cp = fname
  72    else:
  73        try:
  74            cp = configparser.ConfigParser(defaults)
  75            if hasattr(fname, 'readline'):
  76                cp.read_file(fname)
  77            else:
  78                encoding = io.text_encoding(encoding)
  79                cp.read(fname, encoding=encoding)
  80        except configparser.ParsingError as e:
  81            raise RuntimeError(f'{fname} is invalid: {e}')
  82
  83    formatters = _create_formatters(cp)
  84
  85    # critical section
  86    logging._acquireLock()
  87    try:
  88        _clearExistingHandlers()
  89
  90        # Handlers add themselves to logging._handlers
  91        handlers = _install_handlers(cp, formatters)
  92        _install_loggers(cp, handlers, disable_existing_loggers)
  93    finally:
  94        logging._releaseLock()
  95
  96
  97def _resolve(name):
  98    """Resolve a dotted name to a global object."""
  99    name = name.split('.')
 100    used = name.pop(0)
 101    found = __import__(used)
 102    for n in name:
 103        used = used + '.' + n
 104        try:
 105            found = getattr(found, n)
 106        except AttributeError:
 107            __import__(used)
 108            found = getattr(found, n)
 109    return found
 110
 111def _strip_spaces(alist):
 112    return map(str.strip, alist)
 113
 114def _create_formatters(cp):
 115    """Create and return formatters"""
 116    flist = cp["formatters"]["keys"]
 117    if not len(flist):
 118        return {}
 119    flist = flist.split(",")
 120    flist = _strip_spaces(flist)
 121    formatters = {}
 122    for form in flist:
 123        sectname = "formatter_%s" % form
 124        fs = cp.get(sectname, "format", raw=True, fallback=None)
 125        dfs = cp.get(sectname, "datefmt", raw=True, fallback=None)
 126        stl = cp.get(sectname, "style", raw=True, fallback='%')
 127        defaults = cp.get(sectname, "defaults", raw=True, fallback=None)
 128
 129        c = logging.Formatter
 130        class_name = cp[sectname].get("class")
 131        if class_name:
 132            c = _resolve(class_name)
 133
 134        if defaults is not None:
 135            defaults = eval(defaults, vars(logging))
 136            f = c(fs, dfs, stl, defaults=defaults)
 137        else:
 138            f = c(fs, dfs, stl)
 139        formatters[form] = f
 140    return formatters
 141
 142
 143def _install_handlers(cp, formatters):
 144    """Install and return handlers"""
 145    hlist = cp["handlers"]["keys"]
 146    if not len(hlist):
 147        return {}
 148    hlist = hlist.split(",")
 149    hlist = _strip_spaces(hlist)
 150    handlers = {}
 151    fixups = [] #for inter-handler references
 152    for hand in hlist:
 153        section = cp["handler_%s" % hand]
 154        klass = section["class"]
 155        fmt = section.get("formatter", "")
 156        try:
 157            klass = eval(klass, vars(logging))
 158        except (AttributeError, NameError):
 159            klass = _resolve(klass)
 160        args = section.get("args", '()')
 161        args = eval(args, vars(logging))
 162        kwargs = section.get("kwargs", '{}')
 163        kwargs = eval(kwargs, vars(logging))
 164        h = klass(*args, **kwargs)
 165        h.name = hand
 166        if "level" in section:
 167            level = section["level"]
 168            h.setLevel(level)
 169        if len(fmt):
 170            h.setFormatter(formatters[fmt])
 171        if issubclass(klass, logging.handlers.MemoryHandler):
 172            target = section.get("target", "")
 173            if len(target): #the target handler may not be loaded yet, so keep for later...
 174                fixups.append((h, target))
 175        handlers[hand] = h
 176    #now all handlers are loaded, fixup inter-handler references...
 177    for h, t in fixups:
 178        h.setTarget(handlers[t])
 179    return handlers
 180
 181def _handle_existing_loggers(existing, child_loggers, disable_existing):
 182    """
 183    When (re)configuring logging, handle loggers which were in the previous
 184    configuration but are not in the new configuration. There's no point
 185    deleting them as other threads may continue to hold references to them;
 186    and by disabling them, you stop them doing any logging.
 187
 188    However, don't disable children of named loggers, as that's probably not
 189    what was intended by the user. Also, allow existing loggers to NOT be
 190    disabled if disable_existing is false.
 191    """
 192    root = logging.root
 193    for log in existing:
 194        logger = root.manager.loggerDict[log]
 195        if log in child_loggers:
 196            if not isinstance(logger, logging.PlaceHolder):
 197                logger.setLevel(logging.NOTSET)
 198                logger.handlers = []
 199                logger.propagate = True
 200        else:
 201            logger.disabled = disable_existing
 202
 203def _install_loggers(cp, handlers, disable_existing):
 204    """Create and install loggers"""
 205
 206    # configure the root first
 207    llist = cp["loggers"]["keys"]
 208    llist = llist.split(",")
 209    llist = list(_strip_spaces(llist))
 210    llist.remove("root")
 211    section = cp["logger_root"]
 212    root = logging.root
 213    log = root
 214    if "level" in section:
 215        level = section["level"]
 216        log.setLevel(level)
 217    for h in root.handlers[:]:
 218        root.removeHandler(h)
 219    hlist = section["handlers"]
 220    if len(hlist):
 221        hlist = hlist.split(",")
 222        hlist = _strip_spaces(hlist)
 223        for hand in hlist:
 224            log.addHandler(handlers[hand])
 225
 226    #and now the others...
 227    #we don't want to lose the existing loggers,
 228    #since other threads may have pointers to them.
 229    #existing is set to contain all existing loggers,
 230    #and as we go through the new configuration we
 231    #remove any which are configured. At the end,
 232    #what's left in existing is the set of loggers
 233    #which were in the previous configuration but
 234    #which are not in the new configuration.
 235    existing = list(root.manager.loggerDict.keys())
 236    #The list needs to be sorted so that we can
 237    #avoid disabling child loggers of explicitly
 238    #named loggers. With a sorted list it is easier
 239    #to find the child loggers.
 240    existing.sort()
 241    #We'll keep the list of existing loggers
 242    #which are children of named loggers here...
 243    child_loggers = []
 244    #now set up the new ones...
 245    for log in llist:
 246        section = cp["logger_%s" % log]
 247        qn = section["qualname"]
 248        propagate = section.getint("propagate", fallback=1)
 249        logger = logging.getLogger(qn)
 250        if qn in existing:
 251            i = existing.index(qn) + 1 # start with the entry after qn
 252            prefixed = qn + "."
 253            pflen = len(prefixed)
 254            num_existing = len(existing)
 255            while i < num_existing:
 256                if existing[i][:pflen] == prefixed:
 257                    child_loggers.append(existing[i])
 258                i += 1
 259            existing.remove(qn)
 260        if "level" in section:
 261            level = section["level"]
 262            logger.setLevel(level)
 263        for h in logger.handlers[:]:
 264            logger.removeHandler(h)
 265        logger.propagate = propagate
 266        logger.disabled = 0
 267        hlist = section["handlers"]
 268        if len(hlist):
 269            hlist = hlist.split(",")
 270            hlist = _strip_spaces(hlist)
 271            for hand in hlist:
 272                logger.addHandler(handlers[hand])
 273
 274    #Disable any old loggers. There's no point deleting
 275    #them as other threads may continue to hold references
 276    #and by disabling them, you stop them doing any logging.
 277    #However, don't disable children of named loggers, as that's
 278    #probably not what was intended by the user.
 279    #for log in existing:
 280    #    logger = root.manager.loggerDict[log]
 281    #    if log in child_loggers:
 282    #        logger.level = logging.NOTSET
 283    #        logger.handlers = []
 284    #        logger.propagate = 1
 285    #    elif disable_existing_loggers:
 286    #        logger.disabled = 1
 287    _handle_existing_loggers(existing, child_loggers, disable_existing)
 288
 289
 290def _clearExistingHandlers():
 291    """Clear and close existing handlers"""
 292    logging._handlers.clear()
 293    logging.shutdown(logging._handlerList[:])
 294    del logging._handlerList[:]
 295
 296
 297IDENTIFIER = re.compile('^[a-z_][a-z0-9_]*$', re.I)
 298
 299
 300def valid_ident(s):
 301    m = IDENTIFIER.match(s)
 302    if not m:
 303        raise ValueError('Not a valid Python identifier: %r' % s)
 304    return True
 305
 306
 307class ConvertingMixin(object):
 308    """For ConvertingXXX's, this mixin class provides common functions"""
 309
 310    def convert_with_key(self, key, value, replace=True):
 311        result = self.configurator.convert(value)
 312        #If the converted value is different, save for next time
 313        if value is not result:
 314            if replace:
 315                self[key] = result
 316            if type(result) in (ConvertingDict, ConvertingList,
 317                               ConvertingTuple):
 318                result.parent = self
 319                result.key = key
 320        return result
 321
 322    def convert(self, value):
 323        result = self.configurator.convert(value)
 324        if value is not result:
 325            if type(result) in (ConvertingDict, ConvertingList,
 326                               ConvertingTuple):
 327                result.parent = self
 328        return result
 329
 330
 331# The ConvertingXXX classes are wrappers around standard Python containers,
 332# and they serve to convert any suitable values in the container. The
 333# conversion converts base dicts, lists and tuples to their wrapped
 334# equivalents, whereas strings which match a conversion format are converted
 335# appropriately.
 336#
 337# Each wrapper should have a configurator attribute holding the actual
 338# configurator to use for conversion.
 339
 340class ConvertingDict(dict, ConvertingMixin):
 341    """A converting dictionary wrapper."""
 342
 343    def __getitem__(self, key):
 344        value = dict.__getitem__(self, key)
 345        return self.convert_with_key(key, value)
 346
 347    def get(self, key, default=None):
 348        value = dict.get(self, key, default)
 349        return self.convert_with_key(key, value)
 350
 351    def pop(self, key, default=None):
 352        value = dict.pop(self, key, default)
 353        return self.convert_with_key(key, value, replace=False)
 354
 355class ConvertingList(list, ConvertingMixin):
 356    """A converting list wrapper."""
 357    def __getitem__(self, key):
 358        value = list.__getitem__(self, key)
 359        return self.convert_with_key(key, value)
 360
 361    def pop(self, idx=-1):
 362        value = list.pop(self, idx)
 363        return self.convert(value)
 364
 365class ConvertingTuple(tuple, ConvertingMixin):
 366    """A converting tuple wrapper."""
 367    def __getitem__(self, key):
 368        value = tuple.__getitem__(self, key)
 369        # Can't replace a tuple entry.
 370        return self.convert_with_key(key, value, replace=False)
 371
 372class BaseConfigurator(object):
 373    """
 374    The configurator base class which defines some useful defaults.
 375    """
 376
 377    CONVERT_PATTERN = re.compile(r'^(?P<prefix>[a-z]+)://(?P<suffix>.*)$')
 378
 379    WORD_PATTERN = re.compile(r'^\s*(\w+)\s*')
 380    DOT_PATTERN = re.compile(r'^\.\s*(\w+)\s*')
 381    INDEX_PATTERN = re.compile(r'^\[\s*(\w+)\s*\]\s*')
 382    DIGIT_PATTERN = re.compile(r'^\d+$')
 383
 384    value_converters = {
 385        'ext' : 'ext_convert',
 386        'cfg' : 'cfg_convert',
 387    }
 388
 389    # We might want to use a different one, e.g. importlib
 390    importer = staticmethod(__import__)
 391
 392    def __init__(self, config):
 393        self.config = ConvertingDict(config)
 394        self.config.configurator = self
 395
 396    def resolve(self, s):
 397        """
 398        Resolve strings to objects using standard import and attribute
 399        syntax.
 400        """
 401        name = s.split('.')
 402        used = name.pop(0)
 403        try:
 404            found = self.importer(used)
 405            for frag in name:
 406                used += '.' + frag
 407                try:
 408                    found = getattr(found, frag)
 409                except AttributeError:
 410                    self.importer(used)
 411                    found = getattr(found, frag)
 412            return found
 413        except ImportError as e:
 414            v = ValueError('Cannot resolve %r: %s' % (s, e))
 415            raise v from e
 416
 417    def ext_convert(self, value):
 418        """Default converter for the ext:// protocol."""
 419        return self.resolve(value)
 420
 421    def cfg_convert(self, value):
 422        """Default converter for the cfg:// protocol."""
 423        rest = value
 424        m = self.WORD_PATTERN.match(rest)
 425        if m is None:
 426            raise ValueError("Unable to convert %r" % value)
 427        else:
 428            rest = rest[m.end():]
 429            d = self.config[m.groups()[0]]
 430            #print d, rest
 431            while rest:
 432                m = self.DOT_PATTERN.match(rest)
 433                if m:
 434                    d = d[m.groups()[0]]
 435                else:
 436                    m = self.INDEX_PATTERN.match(rest)
 437                    if m:
 438                        idx = m.groups()[0]
 439                        if not self.DIGIT_PATTERN.match(idx):
 440                            d = d[idx]
 441                        else:
 442                            try:
 443                                n = int(idx) # try as number first (most likely)
 444                                d = d[n]
 445                            except TypeError:
 446                                d = d[idx]
 447                if m:
 448                    rest = rest[m.end():]
 449                else:
 450                    raise ValueError('Unable to convert '
 451                                     '%r at %r' % (value, rest))
 452        #rest should be empty
 453        return d
 454
 455    def convert(self, value):
 456        """
 457        Convert values to an appropriate type. dicts, lists and tuples are
 458        replaced by their converting alternatives. Strings are checked to
 459        see if they have a conversion format and are converted if they do.
 460        """
 461        if not isinstance(value, ConvertingDict) and isinstance(value, dict):
 462            value = ConvertingDict(value)
 463            value.configurator = self
 464        elif not isinstance(value, ConvertingList) and isinstance(value, list):
 465            value = ConvertingList(value)
 466            value.configurator = self
 467        elif not isinstance(value, ConvertingTuple) and\
 468                 isinstance(value, tuple) and not hasattr(value, '_fields'):
 469            value = ConvertingTuple(value)
 470            value.configurator = self
 471        elif isinstance(value, str): # str for py3k
 472            m = self.CONVERT_PATTERN.match(value)
 473            if m:
 474                d = m.groupdict()
 475                prefix = d['prefix']
 476                converter = self.value_converters.get(prefix, None)
 477                if converter:
 478                    suffix = d['suffix']
 479                    converter = getattr(self, converter)
 480                    value = converter(suffix)
 481        return value
 482
 483    def configure_custom(self, config):
 484        """Configure an object with a user-supplied factory."""
 485        c = config.pop('()')
 486        if not callable(c):
 487            c = self.resolve(c)
 488        # Check for valid identifiers
 489        kwargs = {k: config[k] for k in config if (k != '.' and valid_ident(k))}
 490        result = c(**kwargs)
 491        props = config.pop('.', None)
 492        if props:
 493            for name, value in props.items():
 494                setattr(result, name, value)
 495        return result
 496
 497    def as_tuple(self, value):
 498        """Utility function which converts lists to tuples."""
 499        if isinstance(value, list):
 500            value = tuple(value)
 501        return value
 502
 503def _is_queue_like_object(obj):
 504    """Check that *obj* implements the Queue API."""
 505    if isinstance(obj, queue.Queue):
 506        return True
 507    # defer importing multiprocessing as much as possible
 508    from multiprocessing.queues import Queue as MPQueue
 509    if isinstance(obj, MPQueue):
 510        return True
 511    # Depending on the multiprocessing start context, we cannot create
 512    # a multiprocessing.managers.BaseManager instance 'mm' to get the
 513    # runtime type of mm.Queue() or mm.JoinableQueue() (see gh-119819).
 514    #
 515    # Since we only need an object implementing the Queue API, we only
 516    # do a protocol check, but we do not use typing.runtime_checkable()
 517    # and typing.Protocol to reduce import time (see gh-121723).
 518    #
 519    # Ideally, we would have wanted to simply use strict type checking
 520    # instead of a protocol-based type checking since the latter does
 521    # not check the method signatures.
 522    queue_interface = [
 523        'empty', 'full', 'get', 'get_nowait',
 524        'put', 'put_nowait', 'join', 'qsize',
 525        'task_done',
 526    ]
 527    return all(callable(getattr(obj, method, None))
 528               for method in queue_interface)
 529
 530class DictConfigurator(BaseConfigurator):
 531    """
 532    Configure logging using a dictionary-like object to describe the
 533    configuration.
 534    """
 535
 536    def configure(self):
 537        """Do the configuration."""
 538
 539        config = self.config
 540        if 'version' not in config:
 541            raise ValueError("dictionary doesn't specify a version")
 542        if config['version'] != 1:
 543            raise ValueError("Unsupported version: %s" % config['version'])
 544        incremental = config.pop('incremental', False)
 545        EMPTY_DICT = {}
 546        logging._acquireLock()
 547        try:
 548            if incremental:
 549                handlers = config.get('handlers', EMPTY_DICT)
 550                for name in handlers:
 551                    if name not in logging._handlers:
 552                        raise ValueError('No handler found with '
 553                                         'name %r'  % name)
 554                    else:
 555                        try:
 556                            handler = logging._handlers[name]
 557                            handler_config = handlers[name]
 558                            level = handler_config.get('level', None)
 559                            if level:
 560                                handler.setLevel(logging._checkLevel(level))
 561                        except Exception as e:
 562                            raise ValueError('Unable to configure handler '
 563                                             '%r' % name) from e
 564                loggers = config.get('loggers', EMPTY_DICT)
 565                for name in loggers:
 566                    try:
 567                        self.configure_logger(name, loggers[name], True)
 568                    except Exception as e:
 569                        raise ValueError('Unable to configure logger '
 570                                         '%r' % name) from e
 571                root = config.get('root', None)
 572                if root:
 573                    try:
 574                        self.configure_root(root, True)
 575                    except Exception as e:
 576                        raise ValueError('Unable to configure root '
 577                                         'logger') from e
 578            else:
 579                disable_existing = config.pop('disable_existing_loggers', True)
 580
 581                _clearExistingHandlers()
 582
 583                # Do formatters first - they don't refer to anything else
 584                formatters = config.get('formatters', EMPTY_DICT)
 585                for name in formatters:
 586                    try:
 587                        formatters[name] = self.configure_formatter(
 588                                                            formatters[name])
 589                    except Exception as e:
 590                        raise ValueError('Unable to configure '
 591                                         'formatter %r' % name) from e
 592                # Next, do filters - they don't refer to anything else, either
 593                filters = config.get('filters', EMPTY_DICT)
 594                for name in filters:
 595                    try:
 596                        filters[name] = self.configure_filter(filters[name])
 597                    except Exception as e:
 598                        raise ValueError('Unable to configure '
 599                                         'filter %r' % name) from e
 600
 601                # Next, do handlers - they refer to formatters and filters
 602                # As handlers can refer to other handlers, sort the keys
 603                # to allow a deterministic order of configuration
 604                handlers = config.get('handlers', EMPTY_DICT)
 605                deferred = []
 606                for name in sorted(handlers):
 607                    try:
 608                        handler = self.configure_handler(handlers[name])
 609                        handler.name = name
 610                        handlers[name] = handler
 611                    except Exception as e:
 612                        if ' not configured yet' in str(e.__cause__):
 613                            deferred.append(name)
 614                        else:
 615                            raise ValueError('Unable to configure handler '
 616                                             '%r' % name) from e
 617
 618                # Now do any that were deferred
 619                for name in deferred:
 620                    try:
 621                        handler = self.configure_handler(handlers[name])
 622                        handler.name = name
 623                        handlers[name] = handler
 624                    except Exception as e:
 625                        raise ValueError('Unable to configure handler '
 626                                         '%r' % name) from e
 627
 628                # Next, do loggers - they refer to handlers and filters
 629
 630                #we don't want to lose the existing loggers,
 631                #since other threads may have pointers to them.
 632                #existing is set to contain all existing loggers,
 633                #and as we go through the new configuration we
 634                #remove any which are configured. At the end,
 635                #what's left in existing is the set of loggers
 636                #which were in the previous configuration but
 637                #which are not in the new configuration.
 638                root = logging.root
 639                existing = list(root.manager.loggerDict.keys())
 640                #The list needs to be sorted so that we can
 641                #avoid disabling child loggers of explicitly
 642                #named loggers. With a sorted list it is easier
 643                #to find the child loggers.
 644                existing.sort()
 645                #We'll keep the list of existing loggers
 646                #which are children of named loggers here...
 647                child_loggers = []
 648                #now set up the new ones...
 649                loggers = config.get('loggers', EMPTY_DICT)
 650                for name in loggers:
 651                    if name in existing:
 652                        i = existing.index(name) + 1 # look after name
 653                        prefixed = name + "."
 654                        pflen = len(prefixed)
 655                        num_existing = len(existing)
 656                        while i < num_existing:
 657                            if existing[i][:pflen] == prefixed:
 658                                child_loggers.append(existing[i])
 659                            i += 1
 660                        existing.remove(name)
 661                    try:
 662                        self.configure_logger(name, loggers[name])
 663                    except Exception as e:
 664                        raise ValueError('Unable to configure logger '
 665                                         '%r' % name) from e
 666
 667                #Disable any old loggers. There's no point deleting
 668                #them as other threads may continue to hold references
 669                #and by disabling them, you stop them doing any logging.
 670                #However, don't disable children of named loggers, as that's
 671                #probably not what was intended by the user.
 672                #for log in existing:
 673                #    logger = root.manager.loggerDict[log]
 674                #    if log in child_loggers:
 675                #        logger.level = logging.NOTSET
 676                #        logger.handlers = []
 677                #        logger.propagate = True
 678                #    elif disable_existing:
 679                #        logger.disabled = True
 680                _handle_existing_loggers(existing, child_loggers,
 681                                         disable_existing)
 682
 683                # And finally, do the root logger
 684                root = config.get('root', None)
 685                if root:
 686                    try:
 687                        self.configure_root(root)
 688                    except Exception as e:
 689                        raise ValueError('Unable to configure root '
 690                                         'logger') from e
 691        finally:
 692            logging._releaseLock()
 693
 694    def configure_formatter(self, config):
 695        """Configure a formatter from a dictionary."""
 696        if '()' in config:
 697            factory = config['()'] # for use in exception handler
 698            try:
 699                result = self.configure_custom(config)
 700            except TypeError as te:
 701                if "'format'" not in str(te):
 702                    raise
 703                #Name of parameter changed from fmt to format.
 704                #Retry with old name.
 705                #This is so that code can be used with older Python versions
 706                #(e.g. by Django)
 707                config['fmt'] = config.pop('format')
 708                config['()'] = factory
 709                result = self.configure_custom(config)
 710        else:
 711            fmt = config.get('format', None)
 712            dfmt = config.get('datefmt', None)
 713            style = config.get('style', '%')
 714            cname = config.get('class', None)
 715            defaults = config.get('defaults', None)
 716
 717            if not cname:
 718                c = logging.Formatter
 719            else:
 720                c = _resolve(cname)
 721
 722            kwargs  = {}
 723
 724            # Add defaults only if it exists.
 725            # Prevents TypeError in custom formatter callables that do not
 726            # accept it.
 727            if defaults is not None:
 728                kwargs['defaults'] = defaults
 729
 730            # A TypeError would be raised if "validate" key is passed in with a formatter callable
 731            # that does not accept "validate" as a parameter
 732            if 'validate' in config:  # if user hasn't mentioned it, the default will be fine
 733                result = c(fmt, dfmt, style, config['validate'], **kwargs)
 734            else:
 735                result = c(fmt, dfmt, style, **kwargs)
 736
 737        return result
 738
 739    def configure_filter(self, config):
 740        """Configure a filter from a dictionary."""
 741        if '()' in config:
 742            result = self.configure_custom(config)
 743        else:
 744            name = config.get('name', '')
 745            result = logging.Filter(name)
 746        return result
 747
 748    def add_filters(self, filterer, filters):
 749        """Add filters to a filterer from a list of names."""
 750        for f in filters:
 751            try:
 752                if callable(f) or callable(getattr(f, 'filter', None)):
 753                    filter_ = f
 754                else:
 755                    filter_ = self.config['filters'][f]
 756                filterer.addFilter(filter_)
 757            except Exception as e:
 758                raise ValueError('Unable to add filter %r' % f) from e
 759
 760    def _configure_queue_handler(self, klass, **kwargs):
 761        if 'queue' in kwargs:
 762            q = kwargs.pop('queue')
 763        else:
 764            q = queue.Queue()  # unbounded
 765
 766        rhl = kwargs.pop('respect_handler_level', False)
 767        lklass = kwargs.pop('listener', logging.handlers.QueueListener)
 768        handlers = kwargs.pop('handlers', [])
 769
 770        listener = lklass(q, *handlers, respect_handler_level=rhl)
 771        handler = klass(q, **kwargs)
 772        handler.listener = listener
 773        return handler
 774
 775    def configure_handler(self, config):
 776        """Configure a handler from a dictionary."""
 777        config_copy = dict(config)  # for restoring in case of error
 778        formatter = config.pop('formatter', None)
 779        if formatter:
 780            try:
 781                formatter = self.config['formatters'][formatter]
 782            except Exception as e:
 783                raise ValueError('Unable to set formatter '
 784                                 '%r' % formatter) from e
 785        level = config.pop('level', None)
 786        filters = config.pop('filters', None)
 787        if '()' in config:
 788            c = config.pop('()')
 789            if not callable(c):
 790                c = self.resolve(c)
 791            factory = c
 792        else:
 793            cname = config.pop('class')
 794            if callable(cname):
 795                klass = cname
 796            else:
 797                klass = self.resolve(cname)
 798            if issubclass(klass, logging.handlers.MemoryHandler):
 799                if 'flushLevel' in config:
 800                    config['flushLevel'] = logging._checkLevel(config['flushLevel'])
 801                if 'target' in config:
 802                    # Special case for handler which refers to another handler
 803                    try:
 804                        tn = config['target']
 805                        th = self.config['handlers'][tn]
 806                        if not isinstance(th, logging.Handler):
 807                            config.update(config_copy)  # restore for deferred cfg
 808                            raise TypeError('target not configured yet')
 809                        config['target'] = th
 810                    except Exception as e:
 811                        raise ValueError('Unable to set target handler %r' % tn) from e
 812            elif issubclass(klass, logging.handlers.QueueHandler):
 813                # Another special case for handler which refers to other handlers
 814                # if 'handlers' not in config:
 815                    # raise ValueError('No handlers specified for a QueueHandler')
 816                if 'queue' in config:
 817                    qspec = config['queue']
 818
 819                    if isinstance(qspec, str):
 820                        q = self.resolve(qspec)
 821                        if not callable(q):
 822                            raise TypeError('Invalid queue specifier %r' % qspec)
 823                        config['queue'] = q()
 824                    elif isinstance(qspec, dict):
 825                        if '()' not in qspec:
 826                            raise TypeError('Invalid queue specifier %r' % qspec)
 827                        config['queue'] = self.configure_custom(dict(qspec))
 828                    elif not _is_queue_like_object(qspec):
 829                        raise TypeError('Invalid queue specifier %r' % qspec)
 830
 831                if 'listener' in config:
 832                    lspec = config['listener']
 833                    if isinstance(lspec, type):
 834                        if not issubclass(lspec, logging.handlers.QueueListener):
 835                            raise TypeError('Invalid listener specifier %r' % lspec)
 836                    else:
 837                        if isinstance(lspec, str):
 838                            listener = self.resolve(lspec)
 839                            if isinstance(listener, type) and\
 840                                not issubclass(listener, logging.handlers.QueueListener):
 841                                raise TypeError('Invalid listener specifier %r' % lspec)
 842                        elif isinstance(lspec, dict):
 843                            if '()' not in lspec:
 844                                raise TypeError('Invalid listener specifier %r' % lspec)
 845                            listener = self.configure_custom(dict(lspec))
 846                        else:
 847                            raise TypeError('Invalid listener specifier %r' % lspec)
 848                        if not callable(listener):
 849                            raise TypeError('Invalid listener specifier %r' % lspec)
 850                        config['listener'] = listener
 851                if 'handlers' in config:
 852                    hlist = []
 853                    try:
 854                        for hn in config['handlers']:
 855                            h = self.config['handlers'][hn]
 856                            if not isinstance(h, logging.Handler):
 857                                config.update(config_copy)  # restore for deferred cfg
 858                                raise TypeError('Required handler %r '
 859                                                'is not configured yet' % hn)
 860                            hlist.append(h)
 861                    except Exception as e:
 862                        raise ValueError('Unable to set required handler %r' % hn) from e
 863                    config['handlers'] = hlist
 864            elif issubclass(klass, logging.handlers.SMTPHandler) and\
 865                'mailhost' in config:
 866                config['mailhost'] = self.as_tuple(config['mailhost'])
 867            elif issubclass(klass, logging.handlers.SysLogHandler) and\
 868                'address' in config:
 869                config['address'] = self.as_tuple(config['address'])
 870            if issubclass(klass, logging.handlers.QueueHandler):
 871                factory = functools.partial(self._configure_queue_handler, klass)
 872            else:
 873                factory = klass
 874        kwargs = {k: config[k] for k in config if (k != '.' and valid_ident(k))}
 875        try:
 876            result = factory(**kwargs)
 877        except TypeError as te:
 878            if "'stream'" not in str(te):
 879                raise
 880            #The argument name changed from strm to stream
 881            #Retry with old name.
 882            #This is so that code can be used with older Python versions
 883            #(e.g. by Django)
 884            kwargs['strm'] = kwargs.pop('stream')
 885            result = factory(**kwargs)
 886        if formatter:
 887            result.setFormatter(formatter)
 888        if level is not None:
 889            result.setLevel(logging._checkLevel(level))
 890        if filters:
 891            self.add_filters(result, filters)
 892        props = config.pop('.', None)
 893        if props:
 894            for name, value in props.items():
 895                setattr(result, name, value)
 896        return result
 897
 898    def add_handlers(self, logger, handlers):
 899        """Add handlers to a logger from a list of names."""
 900        for h in handlers:
 901            try:
 902                logger.addHandler(self.config['handlers'][h])
 903            except Exception as e:
 904                raise ValueError('Unable to add handler %r' % h) from e
 905
 906    def common_logger_config(self, logger, config, incremental=False):
 907        """
 908        Perform configuration which is common to root and non-root loggers.
 909        """
 910        level = config.get('level', None)
 911        if level is not None:
 912            logger.setLevel(logging._checkLevel(level))
 913        if not incremental:
 914            #Remove any existing handlers
 915            for h in logger.handlers[:]:
 916                logger.removeHandler(h)
 917            handlers = config.get('handlers', None)
 918            if handlers:
 919                self.add_handlers(logger, handlers)
 920            filters = config.get('filters', None)
 921            if filters:
 922                self.add_filters(logger, filters)
 923
 924    def configure_logger(self, name, config, incremental=False):
 925        """Configure a non-root logger from a dictionary."""
 926        logger = logging.getLogger(name)
 927        self.common_logger_config(logger, config, incremental)
 928        logger.disabled = False
 929        propagate = config.get('propagate', None)
 930        if propagate is not None:
 931            logger.propagate = propagate
 932
 933    def configure_root(self, config, incremental=False):
 934        """Configure a root logger from a dictionary."""
 935        root = logging.getLogger()
 936        self.common_logger_config(root, config, incremental)
 937
 938dictConfigClass = DictConfigurator
 939
 940def dictConfig(config):
 941    """Configure logging using a dictionary."""
 942    dictConfigClass(config).configure()
 943
 944
 945def listen(port=DEFAULT_LOGGING_CONFIG_PORT, verify=None):
 946    """
 947    Start up a socket server on the specified port, and listen for new
 948    configurations.
 949
 950    These will be sent as a file suitable for processing by fileConfig().
 951    Returns a Thread object on which you can call start() to start the server,
 952    and which you can join() when appropriate. To stop the server, call
 953    stopListening().
 954
 955    Use the ``verify`` argument to verify any bytes received across the wire
 956    from a client. If specified, it should be a callable which receives a
 957    single argument - the bytes of configuration data received across the
 958    network - and it should return either ``None``, to indicate that the
 959    passed in bytes could not be verified and should be discarded, or a
 960    byte string which is then passed to the configuration machinery as
 961    normal. Note that you can return transformed bytes, e.g. by decrypting
 962    the bytes passed in.
 963    """
 964
 965    class ConfigStreamHandler(StreamRequestHandler):
 966        """
 967        Handler for a logging configuration request.
 968
 969        It expects a completely new logging configuration and uses fileConfig
 970        to install it.
 971        """
 972        def handle(self):
 973            """
 974            Handle a request.
 975
 976            Each request is expected to be a 4-byte length, packed using
 977            struct.pack(">L", n), followed by the config file.
 978            Uses fileConfig() to do the grunt work.
 979            """
 980            try:
 981                conn = self.connection
 982                chunk = conn.recv(4)
 983                if len(chunk) == 4:
 984                    slen = struct.unpack(">L", chunk)[0]
 985                    chunk = self.connection.recv(slen)
 986                    while len(chunk) < slen:
 987                        chunk = chunk + conn.recv(slen - len(chunk))
 988                    if self.server.verify is not None:
 989                        chunk = self.server.verify(chunk)
 990                    if chunk is not None:   # verified, can process
 991                        chunk = chunk.decode("utf-8")
 992                        try:
 993                            import json
 994                            d =json.loads(chunk)
 995                            assert isinstance(d, dict)
 996                            dictConfig(d)
 997                        except Exception:
 998                            #Apply new configuration.
 999
1000                            file = io.StringIO(chunk)
1001                            try:
1002                                fileConfig(file)
1003                            except Exception:
1004                                traceback.print_exc()
1005                    if self.server.ready:
1006                        self.server.ready.set()
1007            except OSError as e:
1008                if e.errno != RESET_ERROR:
1009                    raise
1010
1011    class ConfigSocketReceiver(ThreadingTCPServer):
1012        """
1013        A simple TCP socket-based logging config receiver.
1014        """
1015
1016        allow_reuse_address = 1
1017
1018        def __init__(self, host='localhost', port=DEFAULT_LOGGING_CONFIG_PORT,
1019                     handler=None, ready=None, verify=None):
1020            ThreadingTCPServer.__init__(self, (host, port), handler)
1021            logging._acquireLock()
1022            self.abort = 0
1023            logging._releaseLock()
1024            self.timeout = 1
1025            self.ready = ready
1026            self.verify = verify
1027
1028        def serve_until_stopped(self):
1029            import select
1030            abort = 0
1031            while not abort:
1032                rd, wr, ex = select.select([self.socket.fileno()],
1033                                           [], [],
1034                                           self.timeout)
1035                if rd:
1036                    self.handle_request()
1037                logging._acquireLock()
1038                abort = self.abort
1039                logging._releaseLock()
1040            self.server_close()
1041
1042    class Server(threading.Thread):
1043
1044        def __init__(self, rcvr, hdlr, port, verify):
1045            super(Server, self).__init__()
1046            self.rcvr = rcvr
1047            self.hdlr = hdlr
1048            self.port = port
1049            self.verify = verify
1050            self.ready = threading.Event()
1051
1052        def run(self):
1053            server = self.rcvr(port=self.port, handler=self.hdlr,
1054                               ready=self.ready,
1055                               verify=self.verify)
1056            if self.port == 0:
1057                self.port = server.server_address[1]
1058            self.ready.set()
1059            global _listener
1060            logging._acquireLock()
1061            _listener = server
1062            logging._releaseLock()
1063            server.serve_until_stopped()
1064
1065    return Server(ConfigSocketReceiver, ConfigStreamHandler, port, verify)
1066
1067def stopListening():
1068    """
1069    Stop the listening server which was created with a call to listen().
1070    """
1071    global _listener
1072    logging._acquireLock()
1073    try:
1074        if _listener:
1075            _listener.abort = 1
1076            _listener = None
1077    finally:
1078        logging._releaseLock()